[PATCH v2 33/50] mips: octeon: Add misc remaining header files

Daniel Schwierzeck daniel.schwierzeck at gmail.com
Fri Apr 23 18:38:37 CEST 2021


Hi Stefan,

Am Freitag, den 23.04.2021, 05:56 +0200 schrieb Stefan Roese:
> From: Aaron Williams <awilliams at marvell.com>
> 
> Import misc remaining header files from 2013 U-Boot. These will be
> used
> by the later added drivers to support PCIe and networking on the MIPS
> Octeon II / III platforms.
> 
> Signed-off-by: Aaron Williams <awilliams at marvell.com>
> Signed-off-by: Stefan Roese <sr at denx.de>
> Cc: Aaron Williams <awilliams at marvell.com>
> Cc: Chandrakala Chavva <cchavva at marvell.com>
> Cc: Daniel Schwierzeck <daniel.schwierzeck at gmail.com>
> ---
> v2:
> - Add missing mach/octeon_qlm.h file (forgot to commit it in v1)
> 

the patch didn't show up in patchwork. But when manually applying,
there is still a build error due to missing mach/octeon_fdt.h

>  .../mach-octeon/include/mach/cvmx-address.h   |  209 ++
>  .../mach-octeon/include/mach/cvmx-cmd-queue.h |  441 +++
>  .../mach-octeon/include/mach/cvmx-csr-enums.h |   87 +
>  arch/mips/mach-octeon/include/mach/cvmx-csr.h |   78 +
>  .../mach-octeon/include/mach/cvmx-error.h     |  456 +++
>  arch/mips/mach-octeon/include/mach/cvmx-fpa.h |  217 ++
>  .../mips/mach-octeon/include/mach/cvmx-fpa1.h |  196 ++
>  .../mips/mach-octeon/include/mach/cvmx-fpa3.h |  566 ++++
>  .../include/mach/cvmx-global-resources.h      |  213 ++
>  arch/mips/mach-octeon/include/mach/cvmx-gmx.h |   16 +
>  .../mach-octeon/include/mach/cvmx-hwfau.h     |  606 ++++
>  .../mach-octeon/include/mach/cvmx-hwpko.h     |  570 ++++
>  arch/mips/mach-octeon/include/mach/cvmx-ilk.h |  154 +
>  arch/mips/mach-octeon/include/mach/cvmx-ipd.h |  233 ++
>  .../mach-octeon/include/mach/cvmx-packet.h    |   40 +
>  .../mips/mach-octeon/include/mach/cvmx-pcie.h |  279 ++
>  arch/mips/mach-octeon/include/mach/cvmx-pip.h | 1080 ++++++
>  .../include/mach/cvmx-pki-resources.h         |  157 +
>  arch/mips/mach-octeon/include/mach/cvmx-pki.h |  970 ++++++
>  .../mach/cvmx-pko-internal-ports-range.h      |   43 +
>  .../include/mach/cvmx-pko3-queue.h            |  175 +
>  arch/mips/mach-octeon/include/mach/cvmx-pow.h | 2991
> +++++++++++++++++
>  arch/mips/mach-octeon/include/mach/cvmx-qlm.h |  304 ++
>  .../mach-octeon/include/mach/cvmx-scratch.h   |  113 +
>  arch/mips/mach-octeon/include/mach/cvmx-wqe.h | 1462 ++++++++
>  .../mach-octeon/include/mach/octeon_qlm.h     |  109 +
>  26 files changed, 11765 insertions(+)
>  create mode 100644 arch/mips/mach-octeon/include/mach/cvmx-address.h
>  create mode 100644 arch/mips/mach-octeon/include/mach/cvmx-cmd-
> queue.h
>  create mode 100644 arch/mips/mach-octeon/include/mach/cvmx-csr-
> enums.h
>  create mode 100644 arch/mips/mach-octeon/include/mach/cvmx-csr.h
>  create mode 100644 arch/mips/mach-octeon/include/mach/cvmx-error.h
>  create mode 100644 arch/mips/mach-octeon/include/mach/cvmx-fpa.h
>  create mode 100644 arch/mips/mach-octeon/include/mach/cvmx-fpa1.h
>  create mode 100644 arch/mips/mach-octeon/include/mach/cvmx-fpa3.h
>  create mode 100644 arch/mips/mach-octeon/include/mach/cvmx-global-
> resources.h
>  create mode 100644 arch/mips/mach-octeon/include/mach/cvmx-gmx.h
>  create mode 100644 arch/mips/mach-octeon/include/mach/cvmx-hwfau.h
>  create mode 100644 arch/mips/mach-octeon/include/mach/cvmx-hwpko.h
>  create mode 100644 arch/mips/mach-octeon/include/mach/cvmx-ilk.h
>  create mode 100644 arch/mips/mach-octeon/include/mach/cvmx-ipd.h
>  create mode 100644 arch/mips/mach-octeon/include/mach/cvmx-packet.h
>  create mode 100644 arch/mips/mach-octeon/include/mach/cvmx-pcie.h
>  create mode 100644 arch/mips/mach-octeon/include/mach/cvmx-pip.h
>  create mode 100644 arch/mips/mach-octeon/include/mach/cvmx-pki-
> resources.h
>  create mode 100644 arch/mips/mach-octeon/include/mach/cvmx-pki.h
>  create mode 100644 arch/mips/mach-octeon/include/mach/cvmx-pko-
> internal-ports-range.h
>  create mode 100644 arch/mips/mach-octeon/include/mach/cvmx-pko3-
> queue.h
>  create mode 100644 arch/mips/mach-octeon/include/mach/cvmx-pow.h
>  create mode 100644 arch/mips/mach-octeon/include/mach/cvmx-qlm.h
>  create mode 100644 arch/mips/mach-octeon/include/mach/cvmx-scratch.h
>  create mode 100644 arch/mips/mach-octeon/include/mach/cvmx-wqe.h
>  create mode 100644 arch/mips/mach-octeon/include/mach/octeon_qlm.h
> 
> diff --git a/arch/mips/mach-octeon/include/mach/cvmx-address.h
> b/arch/mips/mach-octeon/include/mach/cvmx-address.h
> new file mode 100644
> index 000000000000..984f574a75bb
> --- /dev/null
> +++ b/arch/mips/mach-octeon/include/mach/cvmx-address.h
> @@ -0,0 +1,209 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +/*
> + * Copyright (C) 2020 Marvell International Ltd.
> + *
> + * Typedefs and defines for working with Octeon physical addresses.
> + */
> +
> +#ifndef __CVMX_ADDRESS_H__
> +#define __CVMX_ADDRESS_H__
> +
> +typedef enum {
> +	CVMX_MIPS_SPACE_XKSEG = 3LL,
> +	CVMX_MIPS_SPACE_XKPHYS = 2LL,
> +	CVMX_MIPS_SPACE_XSSEG = 1LL,
> +	CVMX_MIPS_SPACE_XUSEG = 0LL
> +} cvmx_mips_space_t;
> +
> +typedef enum {
> +	CVMX_MIPS_XKSEG_SPACE_KSEG0 = 0LL,
> +	CVMX_MIPS_XKSEG_SPACE_KSEG1 = 1LL,
> +	CVMX_MIPS_XKSEG_SPACE_SSEG = 2LL,
> +	CVMX_MIPS_XKSEG_SPACE_KSEG3 = 3LL
> +} cvmx_mips_xkseg_space_t;
> +
> +/* decodes <14:13> of a kseg3 window address */
> +typedef enum {
> +	CVMX_ADD_WIN_SCR = 0L,
> +	CVMX_ADD_WIN_DMA = 1L,
> +	CVMX_ADD_WIN_UNUSED = 2L,
> +	CVMX_ADD_WIN_UNUSED2 = 3L
> +} cvmx_add_win_dec_t;
> +
> +/* decode within DMA space */
> +typedef enum {
> +	CVMX_ADD_WIN_DMA_ADD = 0L,
> +	CVMX_ADD_WIN_DMA_SENDMEM = 1L,
> +	/* store data must be normal DRAM memory space address in this
> case */
> +	CVMX_ADD_WIN_DMA_SENDDMA = 2L,
> +	/* see CVMX_ADD_WIN_DMA_SEND_DEC for data contents */
> +	CVMX_ADD_WIN_DMA_SENDIO = 3L,
> +	/* store data must be normal IO space address in this case */
> +	CVMX_ADD_WIN_DMA_SENDSINGLE = 4L,
> +	/* no write buffer data needed/used */
> +} cvmx_add_win_dma_dec_t;
> +
> +/**
> + *   Physical Address Decode
> + *
> + * Octeon-I HW never interprets this X (<39:36> reserved
> + * for future expansion), software should set to 0.
> + *
> + *  - 0x0 XXX0 0000 0000 to      DRAM         Cached
> + *  - 0x0 XXX0 0FFF FFFF
> + *
> + *  - 0x0 XXX0 1000 0000 to      Boot Bus     Uncached  (Converted
> to 0x1 00X0 1000 0000
> + *  - 0x0 XXX0 1FFF FFFF         +
> EJTAG                           to 0x1 00X0 1FFF FFFF)
> + *
> + *  - 0x0 XXX0 2000 0000 to      DRAM         Cached
> + *  - 0x0 XXXF FFFF FFFF
> + *
> + *  - 0x1 00X0 0000 0000 to      Boot Bus     Uncached
> + *  - 0x1 00XF FFFF FFFF
> + *
> + *  - 0x1 01X0 0000 0000 to      Other NCB    Uncached
> + *  - 0x1 FFXF FFFF FFFF         devices
> + *
> + * Decode of all Octeon addresses
> + */
> +typedef union {
> +	u64 u64;
> +	struct {
> +		cvmx_mips_space_t R : 2;
> +		u64 offset : 62;
> +	} sva;
> +
> +	struct {
> +		u64 zeroes : 33;
> +		u64 offset : 31;
> +	} suseg;
> +
> +	struct {
> +		u64 ones : 33;
> +		cvmx_mips_xkseg_space_t sp : 2;
> +		u64 offset : 29;
> +	} sxkseg;
> +
> +	struct {
> +		cvmx_mips_space_t R : 2;
> +		u64 cca : 3;
> +		u64 mbz : 10;
> +		u64 pa : 49;
> +	} sxkphys;
> +
> +	struct {
> +		u64 mbz : 15;
> +		u64 is_io : 1;
> +		u64 did : 8;
> +		u64 unaddr : 4;
> +		u64 offset : 36;
> +	} sphys;
> +
> +	struct {
> +		u64 zeroes : 24;
> +		u64 unaddr : 4;
> +		u64 offset : 36;
> +	} smem;
> +
> +	struct {
> +		u64 mem_region : 2;
> +		u64 mbz : 13;
> +		u64 is_io : 1;
> +		u64 did : 8;
> +		u64 unaddr : 4;
> +		u64 offset : 36;
> +	} sio;
> +
> +	struct {
> +		u64 ones : 49;
> +		cvmx_add_win_dec_t csrdec : 2;
> +		u64 addr : 13;
> +	} sscr;
> +
> +	/* there should only be stores to IOBDMA space, no loads */
> +	struct {
> +		u64 ones : 49;
> +		cvmx_add_win_dec_t csrdec : 2;
> +		u64 unused2 : 3;
> +		cvmx_add_win_dma_dec_t type : 3;
> +		u64 addr : 7;
> +	} sdma;
> +
> +	struct {
> +		u64 didspace : 24;
> +		u64 unused : 40;
> +	} sfilldidspace;
> +} cvmx_addr_t;
> +
> +/* These macros for used by 32 bit applications */
> +
> +#define CVMX_MIPS32_SPACE_KSEG0	     1l
> +#define CVMX_ADD_SEG32(segment, add) (((s32)segment << 31) |
> (s32)(add))
> +
> +/*
> + * Currently all IOs are performed using XKPHYS addressing. Linux
> uses the
> + * CvmMemCtl register to enable XKPHYS addressing to IO space from
> user mode.
> + * Future OSes may need to change the upper bits of IO addresses.
> The
> + * following define controls the upper two bits for all IO addresses
> generated
> + * by the simple executive library
> + */
> +#define CVMX_IO_SEG CVMX_MIPS_SPACE_XKPHYS
> +
> +/* These macros simplify the process of creating common IO addresses
> */
> +#define CVMX_ADD_SEG(segment, add) ((((u64)segment) << 62) | (add))
> +
> +#define CVMX_ADD_IO_SEG(add) (add)
> +
> +#define CVMX_ADDR_DIDSPACE(did)	   (((CVMX_IO_SEG) << 22) |
> ((1ULL) << 8) | (did))
> +#define CVMX_ADDR_DID(did)	   (CVMX_ADDR_DIDSPACE(did) << 40)
> +#define CVMX_FULL_DID(did, subdid) (((did) << 3) | (subdid))
> +
> +/* from include/ncb_rsl_id.v */
> +#define CVMX_OCT_DID_MIS  0ULL /* misc stuff */
> +#define CVMX_OCT_DID_GMX0 1ULL
> +#define CVMX_OCT_DID_GMX1 2ULL
> +#define CVMX_OCT_DID_PCI  3ULL
> +#define CVMX_OCT_DID_KEY  4ULL
> +#define CVMX_OCT_DID_FPA  5ULL
> +#define CVMX_OCT_DID_DFA  6ULL
> +#define CVMX_OCT_DID_ZIP  7ULL
> +#define CVMX_OCT_DID_RNG  8ULL
> +#define CVMX_OCT_DID_IPD  9ULL
> +#define CVMX_OCT_DID_PKT  10ULL
> +#define CVMX_OCT_DID_TIM  11ULL
> +#define CVMX_OCT_DID_TAG  12ULL
> +/* the rest are not on the IO bus */
> +#define CVMX_OCT_DID_L2C  16ULL
> +#define CVMX_OCT_DID_LMC  17ULL
> +#define CVMX_OCT_DID_SPX0 18ULL
> +#define CVMX_OCT_DID_SPX1 19ULL
> +#define CVMX_OCT_DID_PIP  20ULL
> +#define CVMX_OCT_DID_ASX0 22ULL
> +#define CVMX_OCT_DID_ASX1 23ULL
> +#define CVMX_OCT_DID_IOB  30ULL
> +
> +#define CVMX_OCT_DID_PKT_SEND	 CVMX_FULL_DID(CVMX_OCT_DID_PKT
> , 2ULL)
> +#define CVMX_OCT_DID_TAG_SWTAG	 CVMX_FULL_DID(CVMX_OCT_DID_TAG
> , 0ULL)
> +#define CVMX_OCT_DID_TAG_TAG1	 CVMX_FULL_DID(CVMX_OCT_DID_TAG
> , 1ULL)
> +#define CVMX_OCT_DID_TAG_TAG2	 CVMX_FULL_DID(CVMX_OCT_DID_TAG
> , 2ULL)
> +#define CVMX_OCT_DID_TAG_TAG3	 CVMX_FULL_DID(CVMX_OCT_DID_TAG
> , 3ULL)
> +#define CVMX_OCT_DID_TAG_NULL_RD CVMX_FULL_DID(CVMX_OCT_DID_TAG,
> 4ULL)
> +#define CVMX_OCT_DID_TAG_TAG5	 CVMX_FULL_DID(CVMX_OCT_DID_TAG
> , 5ULL)
> +#define CVMX_OCT_DID_TAG_CSR	 CVMX_FULL_DID(CVMX_OCT_DID_TAG, 7ULL)
> +#define CVMX_OCT_DID_FAU_FAI	 CVMX_FULL_DID(CVMX_OCT_DID_IOB, 0ULL)
> +#define CVMX_OCT_DID_TIM_CSR	 CVMX_FULL_DID(CVMX_OCT_DID_TIM, 0ULL)
> +#define CVMX_OCT_DID_KEY_RW	 CVMX_FULL_DID(CVMX_OCT_DID_KEY, 0ULL)
> +#define CVMX_OCT_DID_PCI_6	 CVMX_FULL_DID(CVMX_OCT_DID_PCI, 6ULL)
> +#define CVMX_OCT_DID_MIS_BOO	 CVMX_FULL_DID(CVMX_OCT_DID_MIS, 0ULL)
> +#define CVMX_OCT_DID_PCI_RML	 CVMX_FULL_DID(CVMX_OCT_DID_PCI, 0ULL)
> +#define CVMX_OCT_DID_IPD_CSR	 CVMX_FULL_DID(CVMX_OCT_DID_IPD, 7ULL)
> +#define CVMX_OCT_DID_DFA_CSR	 CVMX_FULL_DID(CVMX_OCT_DID_DFA, 7ULL)
> +#define CVMX_OCT_DID_MIS_CSR	 CVMX_FULL_DID(CVMX_OCT_DID_MIS, 7ULL)
> +#define CVMX_OCT_DID_ZIP_CSR	 CVMX_FULL_DID(CVMX_OCT_DID_ZIP, 0ULL)
> +
> +/* Cast to unsigned long long, mainly for use in printfs. */
> +#define CAST_ULL(v) ((unsigned long long)(v))
> +
> +#define UNMAPPED_PTR(x) ((1ULL << 63) | (x))
> +
> +#endif /* __CVMX_ADDRESS_H__ */
> diff --git a/arch/mips/mach-octeon/include/mach/cvmx-cmd-queue.h
> b/arch/mips/mach-octeon/include/mach/cvmx-cmd-queue.h
> new file mode 100644
> index 000000000000..ddc294348cb4
> --- /dev/null
> +++ b/arch/mips/mach-octeon/include/mach/cvmx-cmd-queue.h
> @@ -0,0 +1,441 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +/*
> + * Copyright (C) 2020 Marvell International Ltd.
> + *
> + * Support functions for managing command queues used for
> + * various hardware blocks.
> + *
> + * The common command queue infrastructure abstracts out the
> + * software necessary for adding to Octeon's chained queue
> + * structures. These structures are used for commands to the
> + * PKO, ZIP, DFA, RAID, HNA, and DMA engine blocks. Although each
> + * hardware unit takes commands and CSRs of different types,
> + * they all use basic linked command buffers to store the
> + * pending request. In general, users of the CVMX API don't
> + * call cvmx-cmd-queue functions directly. Instead the hardware
> + * unit specific wrapper should be used. The wrappers perform
> + * unit specific validation and CSR writes to submit the
> + * commands.
> + *
> + * Even though most software will never directly interact with
> + * cvmx-cmd-queue, knowledge of its internal workings can help
> + * in diagnosing performance problems and help with debugging.
> + *
> + * Command queue pointers are stored in a global named block
> + * called "cvmx_cmd_queues". Except for the PKO queues, each
> + * hardware queue is stored in its own cache line to reduce SMP
> + * contention on spin locks. The PKO queues are stored such that
> + * every 16th queue is next to each other in memory. This scheme
> + * allows for queues being in separate cache lines when there
> + * are low number of queues per port. With 16 queues per port,
> + * the first queue for each port is in the same cache area. The
> + * second queues for each port are in another area, etc. This
> + * allows software to implement very efficient lockless PKO with
> + * 16 queues per port using a minimum of cache lines per core.
> + * All queues for a given core will be isolated in the same
> + * cache area.
> + *
> + * In addition to the memory pointer layout, cvmx-cmd-queue
> + * provides an optimized fair ll/sc locking mechanism for the
> + * queues. The lock uses a "ticket / now serving" model to
> + * maintain fair order on contended locks. In addition, it uses
> + * predicted locking time to limit cache contention. When a core
> + * know it must wait in line for a lock, it spins on the
> + * internal cycle counter to completely eliminate any causes of
> + * bus traffic.
> + */
> +
> +#ifndef __CVMX_CMD_QUEUE_H__
> +#define __CVMX_CMD_QUEUE_H__
> +
> +/**
> + * By default we disable the max depth support. Most programs
> + * don't use it and it slows down the command queue processing
> + * significantly.
> + */
> +#ifndef CVMX_CMD_QUEUE_ENABLE_MAX_DEPTH
> +#define CVMX_CMD_QUEUE_ENABLE_MAX_DEPTH 0
> +#endif
> +
> +/**
> + * Enumeration representing all hardware blocks that use command
> + * queues. Each hardware block has up to 65536 sub identifiers for
> + * multiple command queues. Not all chips support all hardware
> + * units.
> + */
> +typedef enum {
> +	CVMX_CMD_QUEUE_PKO_BASE = 0x00000,
> +#define
> CVMX_CMD_QUEUE_PKO(queue)                                            
>                       \
> +	((cvmx_cmd_queue_id_t)(CVMX_CMD_QUEUE_PKO_BASE + (0xffff &
> (queue))))
> +	CVMX_CMD_QUEUE_ZIP = 0x10000,
> +#define
> CVMX_CMD_QUEUE_ZIP_QUE(queue)                                        
>                       \
> +	((cvmx_cmd_queue_id_t)(CVMX_CMD_QUEUE_ZIP + (0xffff &
> (queue))))
> +	CVMX_CMD_QUEUE_DFA = 0x20000,
> +	CVMX_CMD_QUEUE_RAID = 0x30000,
> +	CVMX_CMD_QUEUE_DMA_BASE = 0x40000,
> +#define
> CVMX_CMD_QUEUE_DMA(queue)                                            
>                       \
> +	((cvmx_cmd_queue_id_t)(CVMX_CMD_QUEUE_DMA_BASE + (0xffff &
> (queue))))
> +	CVMX_CMD_QUEUE_BCH = 0x50000,
> +#define CVMX_CMD_QUEUE_BCH(queue)
> ((cvmx_cmd_queue_id_t)(CVMX_CMD_QUEUE_BCH + (0xffff & (queue))))
> +	CVMX_CMD_QUEUE_HNA = 0x60000,
> +	CVMX_CMD_QUEUE_END = 0x70000,
> +} cvmx_cmd_queue_id_t;
> +
> +#define CVMX_CMD_QUEUE_ZIP3_QUE(node,
> queue)                                                       \
> +	((cvmx_cmd_queue_id_t)((node) << 24 | CVMX_CMD_QUEUE_ZIP |
> (0xffff & (queue))))
> +
> +/**
> + * Command write operations can fail if the command queue needs
> + * a new buffer and the associated FPA pool is empty. It can also
> + * fail if the number of queued command words reaches the maximum
> + * set at initialization.
> + */
> +typedef enum {
> +	CVMX_CMD_QUEUE_SUCCESS = 0,
> +	CVMX_CMD_QUEUE_NO_MEMORY = -1,
> +	CVMX_CMD_QUEUE_FULL = -2,
> +	CVMX_CMD_QUEUE_INVALID_PARAM = -3,
> +	CVMX_CMD_QUEUE_ALREADY_SETUP = -4,
> +} cvmx_cmd_queue_result_t;
> +
> +typedef struct {
> +	/* First 64-bit word: */
> +	u64 fpa_pool : 16;
> +	u64 base_paddr : 48;
> +	s32 index;
> +	u16 max_depth;
> +	u16 pool_size_m1;
> +} __cvmx_cmd_queue_state_t;
> +
> +/**
> + * command-queue locking uses a fair ticket spinlock algo,
> + * with 64-bit tickets for endianness-neutrality and
> + * counter overflow protection.
> + * Lock is free when both counters are of equal value.
> + */
> +typedef struct {
> +	u64 ticket;
> +	u64 now_serving;
> +} __cvmx_cmd_queue_lock_t;
> +
> +/**
> + * @INTERNAL
> + * This structure contains the global state of all command queues.
> + * It is stored in a bootmem named block and shared by all
> + * applications running on Octeon. Tickets are stored in a different
> + * cache line that queue information to reduce the contention on the
> + * ll/sc used to get a ticket. If this is not the case, the update
> + * of queue state causes the ll/sc to fail quite often.
> + */
> +typedef struct {
> +	__cvmx_cmd_queue_lock_t lock[(CVMX_CMD_QUEUE_END >> 16) * 256];
> +	__cvmx_cmd_queue_state_t state[(CVMX_CMD_QUEUE_END >> 16) *
> 256];
> +} __cvmx_cmd_queue_all_state_t;
> +
> +extern __cvmx_cmd_queue_all_state_t
> *__cvmx_cmd_queue_state_ptrs[CVMX_MAX_NODES];
> +
> +/**
> + * @INTERNAL
> + * Internal function to handle the corner cases
> + * of adding command words to a queue when the current
> + * block is getting full.
> + */
> +cvmx_cmd_queue_result_t
> __cvmx_cmd_queue_write_raw(cvmx_cmd_queue_id_t queue_id,
> +						   __cvmx_cmd_queue_sta
> te_t *qptr, int cmd_count,
> +						   const u64 *cmds);
> +
> +/**
> + * Initialize a command queue for use. The initial FPA buffer is
> + * allocated and the hardware unit is configured to point to the
> + * new command queue.
> + *
> + * @param queue_id  Hardware command queue to initialize.
> + * @param max_depth Maximum outstanding commands that can be queued.
> + * @param fpa_pool  FPA pool the command queues should come from.
> + * @param pool_size Size of each buffer in the FPA pool (bytes)
> + *
> + * @return CVMX_CMD_QUEUE_SUCCESS or a failure code
> + */
> +cvmx_cmd_queue_result_t
> cvmx_cmd_queue_initialize(cvmx_cmd_queue_id_t queue_id, int
> max_depth,
> +						  int fpa_pool, int
> pool_size);
> +
> +/**
> + * Shutdown a queue a free it's command buffers to the FPA. The
> + * hardware connected to the queue must be stopped before this
> + * function is called.
> + *
> + * @param queue_id Queue to shutdown
> + *
> + * @return CVMX_CMD_QUEUE_SUCCESS or a failure code
> + */
> +cvmx_cmd_queue_result_t cvmx_cmd_queue_shutdown(cvmx_cmd_queue_id_t
> queue_id);
> +
> +/**
> + * Return the number of command words pending in the queue. This
> + * function may be relatively slow for some hardware units.
> + *
> + * @param queue_id Hardware command queue to query
> + *
> + * @return Number of outstanding commands
> + */
> +int cvmx_cmd_queue_length(cvmx_cmd_queue_id_t queue_id);
> +
> +/**
> + * Return the command buffer to be written to. The purpose of this
> + * function is to allow CVMX routine access to the low level buffer
> + * for initial hardware setup. User applications should not call
> this
> + * function directly.
> + *
> + * @param queue_id Command queue to query
> + *
> + * @return Command buffer or NULL on failure
> + */
> +void *cvmx_cmd_queue_buffer(cvmx_cmd_queue_id_t queue_id);
> +
> +/**
> + * @INTERNAL
> + * Retrieve or allocate command queue state named block
> + */
> +cvmx_cmd_queue_result_t __cvmx_cmd_queue_init_state_ptr(unsigned int
> node);
> +
> +/**
> + * @INTERNAL
> + * Get the index into the state arrays for the supplied queue id.
> + *
> + * @param queue_id Queue ID to get an index for
> + *
> + * @return Index into the state arrays
> + */
> +static inline unsigned int
> __cvmx_cmd_queue_get_index(cvmx_cmd_queue_id_t queue_id)
> +{
> +	/* Warning: This code currently only works with devices that
> have 256
> +	 * queues or less.  Devices with more than 16 queues are laid
> out in
> +	 * memory to allow cores quick access to every 16th queue. This
> reduces
> +	 * cache thrashing when you are running 16 queues per port to
> support
> +	 * lockless operation
> +	 */
> +	unsigned int unit = (queue_id >> 16) & 0xff;
> +	unsigned int q = (queue_id >> 4) & 0xf;
> +	unsigned int core = queue_id & 0xf;
> +
> +	return (unit << 8) | (core << 4) | q;
> +}
> +
> +static inline int __cvmx_cmd_queue_get_node(cvmx_cmd_queue_id_t
> queue_id)
> +{
> +	unsigned int node = queue_id >> 24;
> +	return node;
> +}
> +
> +/**
> + * @INTERNAL
> + * Lock the supplied queue so nobody else is updating it at the same
> + * time as us.
> + *
> + * @param queue_id Queue ID to lock
> + *
> + */
> +static inline void __cvmx_cmd_queue_lock(cvmx_cmd_queue_id_t
> queue_id)
> +{
> +}
> +
> +/**
> + * @INTERNAL
> + * Unlock the queue, flushing all writes.
> + *
> + * @param queue_id Queue ID to lock
> + *
> + */
> +static inline void __cvmx_cmd_queue_unlock(cvmx_cmd_queue_id_t
> queue_id)
> +{
> +	CVMX_SYNCWS; /* nudge out the unlock. */
> +}
> +
> +/**
> + * @INTERNAL
> + * Initialize a command-queue lock to "unlocked" state.
> + */
> +static inline void __cvmx_cmd_queue_lock_init(cvmx_cmd_queue_id_t
> queue_id)
> +{
> +	unsigned int index = __cvmx_cmd_queue_get_index(queue_id);
> +	unsigned int node = __cvmx_cmd_queue_get_node(queue_id);
> +
> +	__cvmx_cmd_queue_state_ptrs[node]->lock[index] =
> (__cvmx_cmd_queue_lock_t){ 0, 0 };
> +	CVMX_SYNCWS;
> +}
> +
> +/**
> + * @INTERNAL
> + * Get the queue state structure for the given queue id
> + *
> + * @param queue_id Queue id to get
> + *
> + * @return Queue structure or NULL on failure
> + */
> +static inline __cvmx_cmd_queue_state_t
> *__cvmx_cmd_queue_get_state(cvmx_cmd_queue_id_t queue_id)
> +{
> +	unsigned int index;
> +	unsigned int node;
> +	__cvmx_cmd_queue_state_t *qptr;
> +
> +	node = __cvmx_cmd_queue_get_node(queue_id);
> +	index = __cvmx_cmd_queue_get_index(queue_id);
> +
> +	if (cvmx_unlikely(!__cvmx_cmd_queue_state_ptrs[node]))
> +		__cvmx_cmd_queue_init_state_ptr(node);
> +
> +	qptr = &__cvmx_cmd_queue_state_ptrs[node]->state[index];
> +	return qptr;
> +}
> +
> +/**
> + * Write an arbitrary number of command words to a command queue.
> + * This is a generic function; the fixed number of command word
> + * functions yield higher performance.
> + *
> + * @param queue_id  Hardware command queue to write to
> + * @param use_locking
> + *                  Use internal locking to ensure exclusive access
> for queue
> + *                  updates. If you don't use this locking you must
> ensure
> + *                  exclusivity some other way. Locking is strongly
> recommended.
> + * @param cmd_count Number of command words to write
> + * @param cmds      Array of commands to write
> + *
> + * @return CVMX_CMD_QUEUE_SUCCESS or a failure code
> + */
> +static inline cvmx_cmd_queue_result_t
> +cvmx_cmd_queue_write(cvmx_cmd_queue_id_t queue_id, bool use_locking,
> int cmd_count, const u64 *cmds)
> +{
> +	cvmx_cmd_queue_result_t ret = CVMX_CMD_QUEUE_SUCCESS;
> +	u64 *cmd_ptr;
> +
> +	__cvmx_cmd_queue_state_t *qptr =
> __cvmx_cmd_queue_get_state(queue_id);
> +
> +	/* Make sure nobody else is updating the same queue */
> +	if (cvmx_likely(use_locking))
> +		__cvmx_cmd_queue_lock(queue_id);
> +
> +	/* Most of the time there is lots of free words in current
> block */
> +	if (cvmx_unlikely((qptr->index + cmd_count) >= qptr-
> >pool_size_m1)) {
> +		/* The rare case when nearing end of block */
> +		ret = __cvmx_cmd_queue_write_raw(queue_id, qptr,
> cmd_count, cmds);
> +	} else {
> +		cmd_ptr = (u64 *)cvmx_phys_to_ptr((u64)qptr-
> >base_paddr);
> +		/* Loop easy for compiler to unroll for the likely case
> */
> +		while (cmd_count > 0) {
> +			cmd_ptr[qptr->index++] = *cmds++;
> +			cmd_count--;
> +		}
> +	}
> +
> +	/* All updates are complete. Release the lock and return */
> +	if (cvmx_likely(use_locking))
> +		__cvmx_cmd_queue_unlock(queue_id);
> +	else
> +		CVMX_SYNCWS;
> +
> +	return ret;
> +}
> +
> +/**
> + * Simple function to write two command words to a command queue.
> + *
> + * @param queue_id Hardware command queue to write to
> + * @param use_locking
> + *                 Use internal locking to ensure exclusive access
> for queue
> + *                 updates. If you don't use this locking you must
> ensure
> + *                 exclusivity some other way. Locking is strongly
> recommended.
> + * @param cmd1     Command
> + * @param cmd2     Command
> + *
> + * @return CVMX_CMD_QUEUE_SUCCESS or a failure code
> + */
> +static inline cvmx_cmd_queue_result_t
> cvmx_cmd_queue_write2(cvmx_cmd_queue_id_t queue_id,
> +							    bool
> use_locking, u64 cmd1, u64 cmd2)
> +{
> +	cvmx_cmd_queue_result_t ret = CVMX_CMD_QUEUE_SUCCESS;
> +	u64 *cmd_ptr;
> +
> +	__cvmx_cmd_queue_state_t *qptr =
> __cvmx_cmd_queue_get_state(queue_id);
> +
> +	/* Make sure nobody else is updating the same queue */
> +	if (cvmx_likely(use_locking))
> +		__cvmx_cmd_queue_lock(queue_id);
> +
> +	if (cvmx_unlikely((qptr->index + 2) >= qptr->pool_size_m1)) {
> +		/* The rare case when nearing end of block */
> +		u64 cmds[2];
> +
> +		cmds[0] = cmd1;
> +		cmds[1] = cmd2;
> +		ret = __cvmx_cmd_queue_write_raw(queue_id, qptr, 2,
> cmds);
> +	} else {
> +		/* Likely case to work fast */
> +		cmd_ptr = (u64 *)cvmx_phys_to_ptr((u64)qptr-
> >base_paddr);
> +		cmd_ptr += qptr->index;
> +		qptr->index += 2;
> +		cmd_ptr[0] = cmd1;
> +		cmd_ptr[1] = cmd2;
> +	}
> +
> +	/* All updates are complete. Release the lock and return */
> +	if (cvmx_likely(use_locking))
> +		__cvmx_cmd_queue_unlock(queue_id);
> +	else
> +		CVMX_SYNCWS;
> +
> +	return ret;
> +}
> +
> +/**
> + * Simple function to write three command words to a command queue.
> + *
> + * @param queue_id Hardware command queue to write to
> + * @param use_locking
> + *                 Use internal locking to ensure exclusive access
> for queue
> + *                 updates. If you don't use this locking you must
> ensure
> + *                 exclusivity some other way. Locking is strongly
> recommended.
> + * @param cmd1     Command
> + * @param cmd2     Command
> + * @param cmd3     Command
> + *
> + * @return CVMX_CMD_QUEUE_SUCCESS or a failure code
> + */
> +static inline cvmx_cmd_queue_result_t
> +cvmx_cmd_queue_write3(cvmx_cmd_queue_id_t queue_id, bool
> use_locking, u64 cmd1, u64 cmd2, u64 cmd3)
> +{
> +	cvmx_cmd_queue_result_t ret = CVMX_CMD_QUEUE_SUCCESS;
> +	__cvmx_cmd_queue_state_t *qptr =
> __cvmx_cmd_queue_get_state(queue_id);
> +	u64 *cmd_ptr;
> +
> +	/* Make sure nobody else is updating the same queue */
> +	if (cvmx_likely(use_locking))
> +		__cvmx_cmd_queue_lock(queue_id);
> +
> +	if (cvmx_unlikely((qptr->index + 3) >= qptr->pool_size_m1)) {
> +		/* Most of the time there is lots of free words in
> current block */
> +		u64 cmds[3];
> +
> +		cmds[0] = cmd1;
> +		cmds[1] = cmd2;
> +		cmds[2] = cmd3;
> +		ret = __cvmx_cmd_queue_write_raw(queue_id, qptr, 3,
> cmds);
> +	} else {
> +		cmd_ptr = (u64 *)cvmx_phys_to_ptr((u64)qptr-
> >base_paddr);
> +		cmd_ptr += qptr->index;
> +		qptr->index += 3;
> +		cmd_ptr[0] = cmd1;
> +		cmd_ptr[1] = cmd2;
> +		cmd_ptr[2] = cmd3;
> +	}
> +
> +	/* All updates are complete. Release the lock and return */
> +	if (cvmx_likely(use_locking))
> +		__cvmx_cmd_queue_unlock(queue_id);
> +	else
> +		CVMX_SYNCWS;
> +
> +	return ret;
> +}
> +
> +#endif /* __CVMX_CMD_QUEUE_H__ */
> diff --git a/arch/mips/mach-octeon/include/mach/cvmx-csr-enums.h
> b/arch/mips/mach-octeon/include/mach/cvmx-csr-enums.h
> new file mode 100644
> index 000000000000..a8625b4228ac
> --- /dev/null
> +++ b/arch/mips/mach-octeon/include/mach/cvmx-csr-enums.h
> @@ -0,0 +1,87 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +/*
> + * Copyright (C) 2020 Marvell International Ltd.
> + *
> + * Definitions for enumerations used with Octeon CSRs.
> + */
> +
> +#ifndef __CVMX_CSR_ENUMS_H__
> +#define __CVMX_CSR_ENUMS_H__
> +
> +typedef enum {
> +	CVMX_IPD_OPC_MODE_STT = 0LL,
> +	CVMX_IPD_OPC_MODE_STF = 1LL,
> +	CVMX_IPD_OPC_MODE_STF1_STT = 2LL,
> +	CVMX_IPD_OPC_MODE_STF2_STT = 3LL
> +} cvmx_ipd_mode_t;
> +
> +/**
> + * Enumeration representing the amount of packet processing
> + * and validation performed by the input hardware.
> + */
> +typedef enum {
> +	CVMX_PIP_PORT_CFG_MODE_NONE = 0ull,
> +	CVMX_PIP_PORT_CFG_MODE_SKIPL2 = 1ull,
> +	CVMX_PIP_PORT_CFG_MODE_SKIPIP = 2ull
> +} cvmx_pip_port_parse_mode_t;
> +
> +/**
> + * This enumeration controls how a QoS watcher matches a packet.
> + *
> + * @deprecated  This enumeration was used with
> cvmx_pip_config_watcher which has
> + *              been deprecated.
> + */
> +typedef enum {
> +	CVMX_PIP_QOS_WATCH_DISABLE = 0ull,
> +	CVMX_PIP_QOS_WATCH_PROTNH = 1ull,
> +	CVMX_PIP_QOS_WATCH_TCP = 2ull,
> +	CVMX_PIP_QOS_WATCH_UDP = 3ull
> +} cvmx_pip_qos_watch_types;
> +
> +/**
> + * This enumeration is used in PIP tag config to control how
> + * POW tags are generated by the hardware.
> + */
> +typedef enum {
> +	CVMX_PIP_TAG_MODE_TUPLE = 0ull,
> +	CVMX_PIP_TAG_MODE_MASK = 1ull,
> +	CVMX_PIP_TAG_MODE_IP_OR_MASK = 2ull,
> +	CVMX_PIP_TAG_MODE_TUPLE_XOR_MASK = 3ull
> +} cvmx_pip_tag_mode_t;
> +
> +/**
> + * Tag type definitions
> + */
> +typedef enum {
> +	CVMX_POW_TAG_TYPE_ORDERED = 0L,
> +	CVMX_POW_TAG_TYPE_ATOMIC = 1L,
> +	CVMX_POW_TAG_TYPE_NULL = 2L,
> +	CVMX_POW_TAG_TYPE_NULL_NULL = 3L
> +} cvmx_pow_tag_type_t;
> +
> +/**
> + * LCR bits 0 and 1 control the number of bits per character. See
> the following table for encodings:
> + *
> + * - 00 = 5 bits (bits 0-4 sent)
> + * - 01 = 6 bits (bits 0-5 sent)
> + * - 10 = 7 bits (bits 0-6 sent)
> + * - 11 = 8 bits (all bits sent)
> + */
> +typedef enum {
> +	CVMX_UART_BITS5 = 0,
> +	CVMX_UART_BITS6 = 1,
> +	CVMX_UART_BITS7 = 2,
> +	CVMX_UART_BITS8 = 3
> +} cvmx_uart_bits_t;
> +
> +typedef enum {
> +	CVMX_UART_IID_NONE = 1,
> +	CVMX_UART_IID_RX_ERROR = 6,
> +	CVMX_UART_IID_RX_DATA = 4,
> +	CVMX_UART_IID_RX_TIMEOUT = 12,
> +	CVMX_UART_IID_TX_EMPTY = 2,
> +	CVMX_UART_IID_MODEM = 0,
> +	CVMX_UART_IID_BUSY = 7
> +} cvmx_uart_iid_t;
> +
> +#endif /* __CVMX_CSR_ENUMS_H__ */
> diff --git a/arch/mips/mach-octeon/include/mach/cvmx-csr.h
> b/arch/mips/mach-octeon/include/mach/cvmx-csr.h
> new file mode 100644
> index 000000000000..730d54bb9278
> --- /dev/null
> +++ b/arch/mips/mach-octeon/include/mach/cvmx-csr.h
> @@ -0,0 +1,78 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +/*
> + * Copyright (C) 2020 Marvell International Ltd.
> + *
> + * Configuration and status register (CSR) address and type
> definitions for
> + * Octoen.
> + */
> +
> +#ifndef __CVMX_CSR_H__
> +#define __CVMX_CSR_H__
> +
> +#include "cvmx-csr-enums.h"
> +#include "cvmx-pip-defs.h"
> +
> +typedef cvmx_pip_prt_cfgx_t cvmx_pip_port_cfg_t;
> +
> +/* The CSRs for bootbus region zero used to be independent of the
> +    other 1-7. As of SDK 1.7.0 these were combined. These macros
> +    are for backwards compactability */
> +#define CVMX_MIO_BOOT_REG_CFG0 CVMX_MIO_BOOT_REG_CFGX(0)
> +#define CVMX_MIO_BOOT_REG_TIM0 CVMX_MIO_BOOT_REG_TIMX(0)
> +
> +/* The CN3XXX and CN58XX chips used to not have a LMC number
> +    passed to the address macros. These are here to supply backwards
> +    compatibility with old code. Code should really use the new
> addresses
> +    with bus arguments for support on other chips */
> +#define CVMX_LMC_BIST_CTL	  CVMX_LMCX_BIST_CTL(0)
> +#define CVMX_LMC_BIST_RESULT	  CVMX_LMCX_BIST_RESULT(0)
> +#define CVMX_LMC_COMP_CTL	  CVMX_LMCX_COMP_CTL(0)
> +#define CVMX_LMC_CTL		  CVMX_LMCX_CTL(0)
> +#define CVMX_LMC_CTL1		  CVMX_LMCX_CTL1(0)
> +#define CVMX_LMC_DCLK_CNT_HI	  CVMX_LMCX_DCLK_CNT_HI(0)
> +#define CVMX_LMC_DCLK_CNT_LO	  CVMX_LMCX_DCLK_CNT_LO(0)
> +#define CVMX_LMC_DCLK_CTL	  CVMX_LMCX_DCLK_CTL(0)
> +#define CVMX_LMC_DDR2_CTL	  CVMX_LMCX_DDR2_CTL(0)
> +#define CVMX_LMC_DELAY_CFG	  CVMX_LMCX_DELAY_CFG(0)
> +#define CVMX_LMC_DLL_CTL	  CVMX_LMCX_DLL_CTL(0)
> +#define CVMX_LMC_DUAL_MEMCFG	  CVMX_LMCX_DUAL_MEMCFG(0)
> +#define CVMX_LMC_ECC_SYND	  CVMX_LMCX_ECC_SYND(0)
> +#define CVMX_LMC_FADR		  CVMX_LMCX_FADR(0)
> +#define CVMX_LMC_IFB_CNT_HI	  CVMX_LMCX_IFB_CNT_HI(0)
> +#define CVMX_LMC_IFB_CNT_LO	  CVMX_LMCX_IFB_CNT_LO(0)
> +#define CVMX_LMC_MEM_CFG0	  CVMX_LMCX_MEM_CFG0(0)
> +#define CVMX_LMC_MEM_CFG1	  CVMX_LMCX_MEM_CFG1(0)
> +#define CVMX_LMC_OPS_CNT_HI	  CVMX_LMCX_OPS_CNT_HI(0)
> +#define CVMX_LMC_OPS_CNT_LO	  CVMX_LMCX_OPS_CNT_LO(0)
> +#define CVMX_LMC_PLL_BWCTL	  CVMX_LMCX_PLL_BWCTL(0)
> +#define CVMX_LMC_PLL_CTL	  CVMX_LMCX_PLL_CTL(0)
> +#define CVMX_LMC_PLL_STATUS	  CVMX_LMCX_PLL_STATUS(0)
> +#define CVMX_LMC_READ_LEVEL_CTL	  CVMX_LMCX_READ_LEVEL_CTL(0)
> +#define CVMX_LMC_READ_LEVEL_DBG	  CVMX_LMCX_READ_LEVEL_DBG(0)
> +#define CVMX_LMC_READ_LEVEL_RANKX CVMX_LMCX_READ_LEVEL_RANKX(0)
> +#define CVMX_LMC_RODT_COMP_CTL	  CVMX_LMCX_RODT_COMP_CTL(0)
> +#define CVMX_LMC_RODT_CTL	  CVMX_LMCX_RODT_CTL(0)
> +#define CVMX_LMC_WODT_CTL	  CVMX_LMCX_WODT_CTL0(0)
> +#define CVMX_LMC_WODT_CTL0	  CVMX_LMCX_WODT_CTL0(0)
> +#define CVMX_LMC_WODT_CTL1	  CVMX_LMCX_WODT_CTL1(0)
> +
> +/* The CN3XXX and CN58XX chips used to not have a TWSI bus number
> +    passed to the address macros. These are here to supply backwards
> +    compatibility with old code. Code should really use the new
> addresses
> +    with bus arguments for support on other chips */
> +#define CVMX_MIO_TWS_INT	 CVMX_MIO_TWSX_INT(0)
> +#define CVMX_MIO_TWS_SW_TWSI	 CVMX_MIO_TWSX_SW_TWSI(0)
> +#define CVMX_MIO_TWS_SW_TWSI_EXT CVMX_MIO_TWSX_SW_TWSI_EXT(0)
> +#define CVMX_MIO_TWS_TWSI_SW	 CVMX_MIO_TWSX_TWSI_SW(0)
> +
> +/* The CN3XXX and CN58XX chips used to not have a SMI/MDIO bus
> number
> +    passed to the address macros. These are here to supply backwards
> +    compatibility with old code. Code should really use the new
> addresses
> +    with bus arguments for support on other chips */
> +#define CVMX_SMI_CLK	CVMX_SMIX_CLK(0)
> +#define CVMX_SMI_CMD	CVMX_SMIX_CMD(0)
> +#define CVMX_SMI_EN	CVMX_SMIX_EN(0)
> +#define CVMX_SMI_RD_DAT CVMX_SMIX_RD_DAT(0)
> +#define CVMX_SMI_WR_DAT CVMX_SMIX_WR_DAT(0)
> +
> +#endif /* __CVMX_CSR_H__ */
> diff --git a/arch/mips/mach-octeon/include/mach/cvmx-error.h
> b/arch/mips/mach-octeon/include/mach/cvmx-error.h
> new file mode 100644
> index 000000000000..9a13ed422484
> --- /dev/null
> +++ b/arch/mips/mach-octeon/include/mach/cvmx-error.h
> @@ -0,0 +1,456 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +/*
> + * Copyright (C) 2020 Marvell International Ltd.
> + *
> + * Interface to the Octeon extended error status.
> + */
> +
> +#ifndef __CVMX_ERROR_H__
> +#define __CVMX_ERROR_H__
> +
> +/**
> + * There are generally many error status bits associated with a
> + * single logical group. The enumeration below is used to
> + * communicate high level groups to the error infastructure so
> + * error status bits can be enable or disabled in large groups.
> + */
> +typedef enum {
> +	CVMX_ERROR_GROUP_INTERNAL,
> +	CVMX_ERROR_GROUP_L2C,
> +	CVMX_ERROR_GROUP_ETHERNET,
> +	CVMX_ERROR_GROUP_MGMT_PORT,
> +	CVMX_ERROR_GROUP_PCI,
> +	CVMX_ERROR_GROUP_SRIO,
> +	CVMX_ERROR_GROUP_USB,
> +	CVMX_ERROR_GROUP_LMC,
> +	CVMX_ERROR_GROUP_ILK,
> +	CVMX_ERROR_GROUP_DFM,
> +	CVMX_ERROR_GROUP_ILA,
> +} cvmx_error_group_t;
> +
> +/**
> + * Flags representing special handling for some error registers.
> + * These flags are passed to cvmx_error_initialize() to control
> + * the handling of bits where the same flags were passed to the
> + * added cvmx_error_info_t.
> + */
> +typedef enum {
> +	CVMX_ERROR_TYPE_NONE = 0,
> +	CVMX_ERROR_TYPE_SBE = 1 << 0,
> +	CVMX_ERROR_TYPE_DBE = 1 << 1,
> +} cvmx_error_type_t;
> +
> +/**
> + * When registering for interest in an error status register, the
> + * type of the register needs to be known by cvmx-error. Most
> + * registers are either IO64 or IO32, but some blocks contain
> + * registers that can't be directly accessed. A good example of
> + * would be PCIe extended error state stored in config space.
> + */
> +typedef enum {
> +	__CVMX_ERROR_REGISTER_NONE,
> +	CVMX_ERROR_REGISTER_IO64,
> +	CVMX_ERROR_REGISTER_IO32,
> +	CVMX_ERROR_REGISTER_PCICONFIG,
> +	CVMX_ERROR_REGISTER_SRIOMAINT,
> +} cvmx_error_register_t;
> +
> +struct cvmx_error_info;
> +/**
> + * Error handling functions must have the following prototype.
> + */
> +typedef int (*cvmx_error_func_t)(const struct cvmx_error_info
> *info);
> +
> +/**
> + * This structure is passed to all error handling functions.
> + */
> +typedef struct cvmx_error_info {
> +	cvmx_error_register_t reg_type;
> +	u64 status_addr;
> +	u64 status_mask;
> +	u64 enable_addr;
> +	u64 enable_mask;
> +	cvmx_error_type_t flags;
> +	cvmx_error_group_t group;
> +	int group_index;
> +	cvmx_error_func_t func;
> +	u64 user_info;
> +	struct {
> +		cvmx_error_register_t reg_type;
> +		u64 status_addr;
> +		u64 status_mask;
> +	} parent;
> +} cvmx_error_info_t;
> +
> +/**
> + * Initialize the error status system. This should be called once
> + * before any other functions are called. This function adds default
> + * handlers for most all error events but does not enable them.
> Later
> + * calls to cvmx_error_enable() are needed.
> + *
> + * @param flags  Optional flags.
> + *
> + * @return Zero on success, negative on failure.
> + */
> +int cvmx_error_initialize(void);
> +
> +/**
> + * Poll the error status registers and call the appropriate error
> + * handlers. This should be called in the RSL interrupt handler
> + * for your application or operating system.
> + *
> + * @return Number of error handlers called. Zero means this call
> + *         found no errors and was spurious.
> + */
> +int cvmx_error_poll(void);
> +
> +/**
> + * Register to be called when an error status bit is set. Most users
> + * will not need to call this function as cvmx_error_initialize()
> + * registers default handlers for most error conditions. This
> function
> + * is normally used to add more handlers without changing the
> existing
> + * handlers.
> + *
> + * @param new_info Information about the handler for a error
> register. The
> + *                 structure passed is copied and can be destroyed
> after the
> + *                 call. All members of the structure must be
> populated, even the
> + *                 parent information.
> + *
> + * @return Zero on success, negative on failure.
> + */
> +int cvmx_error_add(const cvmx_error_info_t *new_info);
> +
> +/**
> + * Remove all handlers for a status register and mask. Normally
> + * this function should not be called. Instead a new handler should
> be
> + * installed to replace the existing handler. In the even that all
> + * reporting of a error bit should be removed, then use this
> + * function.
> + *
> + * @param reg_type Type of the status register to remove
> + * @param status_addr
> + *                 Status register to remove.
> + * @param status_mask
> + *                 All handlers for this status register with this
> mask will be
> + *                 removed.
> + * @param old_info If not NULL, this is filled with information
> about the handler
> + *                 that was removed.
> + *
> + * @return Zero on success, negative on failure (not found).
> + */
> +int cvmx_error_remove(cvmx_error_register_t reg_type, u64
> status_addr, u64 status_mask,
> +		      cvmx_error_info_t *old_info);
> +
> +/**
> + * Change the function and user_info for an existing error status
> + * register. This function should be used to replace the default
> + * handler with an application specific version as needed.
> + *
> + * @param reg_type Type of the status register to change
> + * @param status_addr
> + *                 Status register to change.
> + * @param status_mask
> + *                 All handlers for this status register with this
> mask will be
> + *                 changed.
> + * @param new_func New function to use to handle the error status
> + * @param new_user_info
> + *                 New user info parameter for the function
> + * @param old_func If not NULL, the old function is returned. Useful
> for restoring
> + *                 the old handler.
> + * @param old_user_info
> + *                 If not NULL, the old user info parameter.
> + *
> + * @return Zero on success, negative on failure
> + */
> +int cvmx_error_change_handler(cvmx_error_register_t reg_type, u64
> status_addr, u64 status_mask,
> +			      cvmx_error_func_t new_func, u64
> new_user_info,
> +			      cvmx_error_func_t *old_func, u64
> *old_user_info);
> +
> +/**
> + * Enable all error registers for a logical group. This should be
> + * called whenever a logical group is brought online.
> + *
> + * @param group  Logical group to enable
> + * @param group_index
> + *               Index for the group as defined in the
> cvmx_error_group_t
> + *               comments.
> + *
> + * @return Zero on success, negative on failure.
> + */
> +/*
> + * Rather than conditionalize the calls throughout the executive to
> not enable
> + * interrupts in Uboot, simply make the enable function do nothing
> + */
> +static inline int cvmx_error_enable_group(cvmx_error_group_t group,
> int group_index)
> +{
> +	return 0;
> +}
> +
> +/**
> + * Disable all error registers for a logical group. This should be
> + * called whenever a logical group is brought offline. Many blocks
> + * will report spurious errors when offline unless this function
> + * is called.
> + *
> + * @param group  Logical group to disable
> + * @param group_index
> + *               Index for the group as defined in the
> cvmx_error_group_t
> + *               comments.
> + *
> + * @return Zero on success, negative on failure.
> + */
> +/*
> + * Rather than conditionalize the calls throughout the executive to
> not disable
> + * interrupts in Uboot, simply make the enable function do nothing
> + */
> +static inline int cvmx_error_disable_group(cvmx_error_group_t group,
> int group_index)
> +{
> +	return 0;
> +}
> +
> +/**
> + * Enable all handlers for a specific status register mask.
> + *
> + * @param reg_type Type of the status register
> + * @param status_addr
> + *                 Status register address
> + * @param status_mask
> + *                 All handlers for this status register with this
> mask will be
> + *                 enabled.
> + *
> + * @return Zero on success, negative on failure.
> + */
> +int cvmx_error_enable(cvmx_error_register_t reg_type, u64
> status_addr, u64 status_mask);
> +
> +/**
> + * Disable all handlers for a specific status register and mask.
> + *
> + * @param reg_type Type of the status register
> + * @param status_addr
> + *                 Status register address
> + * @param status_mask
> + *                 All handlers for this status register with this
> mask will be
> + *                 disabled.
> + *
> + * @return Zero on success, negative on failure.
> + */
> +int cvmx_error_disable(cvmx_error_register_t reg_type, u64
> status_addr, u64 status_mask);
> +
> +/**
> + * @INTERNAL
> + * Function for processing non leaf error status registers. This
> function
> + * calls all handlers for this passed register and all children
> linked
> + * to it.
> + *
> + * @param info   Error register to check
> + *
> + * @return Number of error status bits found or zero if no bits were
> set.
> + */
> +int __cvmx_error_decode(const cvmx_error_info_t *info);
> +
> +/**
> + * @INTERNAL
> + * This error bit handler simply prints a message and clears the
> status bit
> + *
> + * @param info   Error register to check
> + *
> + * @return
> + */
> +int __cvmx_error_display(const cvmx_error_info_t *info);
> +
> +/**
> + * Find the handler for a specific status register and mask
> + *
> + * @param status_addr
> + *                Status register address
> + *
> + * @return  Return the handler on success or null on failure.
> + */
> +cvmx_error_info_t *cvmx_error_get_index(u64 status_addr);
> +
> +void __cvmx_install_gmx_error_handler_for_xaui(void);
> +
> +/**
> + * 78xx related
> + */
> +/**
> + * Compare two INTSN values.
> + *
> + * @param key INTSN value to search for
> + * @param data current entry from the searched array
> + *
> + * @return Negative, 0 or positive when respectively key is less
> than,
> + *		equal or greater than data.
> + */
> +int cvmx_error_intsn_cmp(const void *key, const void *data);
> +
> +/**
> + * @INTERNAL
> + *
> + * @param intsn   Interrupt source number to display
> + *
> + * @param node Node number
> + *
> + * @return Zero on success, -1 on error
> + */
> +int cvmx_error_intsn_display_v3(int node, u32 intsn);
> +
> +/**
> + * Initialize the error status system for cn78xx. This should be
> called once
> + * before any other functions are called. This function enables the
> interrupts
> + * described in the array.
> + *
> + * @param node Node number
> + *
> + * @return Zero on success, negative on failure.
> + */
> +int cvmx_error_initialize_cn78xx(int node);
> +
> +/**
> + * Enable interrupt for a specific INTSN.
> + *
> + * @param node Node number
> + * @param intsn Interrupt source number
> + *
> + * @return Zero on success, negative on failure.
> + */
> +int cvmx_error_intsn_enable_v3(int node, u32 intsn);
> +
> +/**
> + * Disable interrupt for a specific INTSN.
> + *
> + * @param node Node number
> + * @param intsn Interrupt source number
> + *
> + * @return Zero on success, negative on failure.
> + */
> +int cvmx_error_intsn_disable_v3(int node, u32 intsn);
> +
> +/**
> + * Clear interrupt for a specific INTSN.
> + *
> + * @param intsn Interrupt source number
> + *
> + * @return Zero on success, negative on failure.
> + */
> +int cvmx_error_intsn_clear_v3(int node, u32 intsn);
> +
> +/**
> + * Enable interrupts for a specific CSR(all the bits/intsn in the
> csr).
> + *
> + * @param node Node number
> + * @param csr_address CSR address
> + *
> + * @return Zero on success, negative on failure.
> + */
> +int cvmx_error_csr_enable_v3(int node, u64 csr_address);
> +
> +/**
> + * Disable interrupts for a specific CSR (all the bits/intsn in the
> csr).
> + *
> + * @param node Node number
> + * @param csr_address CSR address
> + *
> + * @return Zero
> + */
> +int cvmx_error_csr_disable_v3(int node, u64 csr_address);
> +
> +/**
> + * Enable all error registers for a logical group. This should be
> + * called whenever a logical group is brought online.
> + *
> + * @param group  Logical group to enable
> + * @param xipd_port  The IPD port value
> + *
> + * @return Zero.
> + */
> +int cvmx_error_enable_group_v3(cvmx_error_group_t group, int
> xipd_port);
> +
> +/**
> + * Disable all error registers for a logical group.
> + *
> + * @param group  Logical group to enable
> + * @param xipd_port  The IPD port value
> + *
> + * @return Zero.
> + */
> +int cvmx_error_disable_group_v3(cvmx_error_group_t group, int
> xipd_port);
> +
> +/**
> + * Enable all error registers for a specific category in a logical
> group.
> + * This should be called whenever a logical group is brought online.
> + *
> + * @param group  Logical group to enable
> + * @param type   Category in a logical group to enable
> + * @param xipd_port  The IPD port value
> + *
> + * @return Zero.
> + */
> +int cvmx_error_enable_group_type_v3(cvmx_error_group_t group,
> cvmx_error_type_t type,
> +				    int xipd_port);
> +
> +/**
> + * Disable all error registers for a specific category in a logical
> group.
> + * This should be called whenever a logical group is brought online.
> + *
> + * @param group  Logical group to disable
> + * @param type   Category in a logical group to disable
> + * @param xipd_port  The IPD port value
> + *
> + * @return Zero.
> + */
> +int cvmx_error_disable_group_type_v3(cvmx_error_group_t group,
> cvmx_error_type_t type,
> +				     int xipd_port);
> +
> +/**
> + * Clear all error registers for a logical group.
> + *
> + * @param group  Logical group to disable
> + * @param xipd_port  The IPD port value
> + *
> + * @return Zero.
> + */
> +int cvmx_error_clear_group_v3(cvmx_error_group_t group, int
> xipd_port);
> +
> +/**
> + * Enable all error registers for a particular category.
> + *
> + * @param node  CCPI node
> + * @param type  category to enable
> + *
> + *@return Zero.
> + */
> +int cvmx_error_enable_type_v3(int node, cvmx_error_type_t type);
> +
> +/**
> + * Disable all error registers for a particular category.
> + *
> + * @param node  CCPI node
> + * @param type  category to disable
> + *
> + *@return Zero.
> + */
> +int cvmx_error_disable_type_v3(int node, cvmx_error_type_t type);
> +
> +void cvmx_octeon_hang(void) __attribute__((__noreturn__));
> +
> +/**
> + * @INTERNAL
> + *
> + * Process L2C single and multi-bit ECC errors
> + *
> + */
> +int __cvmx_cn7xxx_l2c_l2d_ecc_error_display(int node, int intsn);
> +
> +/**
> + * Handle L2 cache TAG ECC errors and noway errors
> + *
> + * @param	CCPI node
> + * @param	intsn	intsn from error array.
> + * @param	remote	true for remote node (cn78xx only)
> + *
> + * @return	1 if handled, 0 if not handled
> + */
> +int __cvmx_cn7xxx_l2c_tag_error_display(int node, int intsn, bool
> remote);
> +
> +#endif
> diff --git a/arch/mips/mach-octeon/include/mach/cvmx-fpa.h
> b/arch/mips/mach-octeon/include/mach/cvmx-fpa.h
> new file mode 100644
> index 000000000000..297fb3f4a28c
> --- /dev/null
> +++ b/arch/mips/mach-octeon/include/mach/cvmx-fpa.h
> @@ -0,0 +1,217 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +/*
> + * Copyright (C) 2020 Marvell International Ltd.
> + *
> + * Interface to the hardware Free Pool Allocator.
> + */
> +
> +#ifndef __CVMX_FPA_H__
> +#define __CVMX_FPA_H__
> +
> +#include "cvmx-scratch.h"
> +#include "cvmx-fpa-defs.h"
> +#include "cvmx-fpa1.h"
> +#include "cvmx-fpa3.h"
> +
> +#define CVMX_FPA_MIN_BLOCK_SIZE 128
> +#define CVMX_FPA_ALIGNMENT	128
> +#define CVMX_FPA_POOL_NAME_LEN	16
> +
> +/* On CN78XX in backward-compatible mode, pool is mapped to AURA */
> +#define
> CVMX_FPA_NUM_POOLS                                                   
>                       \
> +	(octeon_has_feature(OCTEON_FEATURE_FPA3) ?
> cvmx_fpa3_num_auras() : CVMX_FPA1_NUM_POOLS)
> +
> +/**
> + * Structure to store FPA pool configuration parameters.
> + */
> +struct cvmx_fpa_pool_config {
> +	s64 pool_num;
> +	u64 buffer_size;
> +	u64 buffer_count;
> +};
> +
> +typedef struct cvmx_fpa_pool_config cvmx_fpa_pool_config_t;
> +
> +/**
> + * Return the name of the pool
> + *
> + * @param pool_num   Pool to get the name of
> + * @return The name
> + */
> +const char *cvmx_fpa_get_name(int pool_num);
> +
> +/**
> + * Initialize FPA per node
> + */
> +int cvmx_fpa_global_init_node(int node);
> +
> +/**
> + * Enable the FPA
> + */
> +static inline void cvmx_fpa_enable(void)
> +{
> +	if (!octeon_has_feature(OCTEON_FEATURE_FPA3))
> +		cvmx_fpa1_enable();
> +	else
> +		cvmx_fpa_global_init_node(cvmx_get_node_num());
> +}
> +
> +/**
> + * Disable the FPA
> + */
> +static inline void cvmx_fpa_disable(void)
> +{
> +	if (!octeon_has_feature(OCTEON_FEATURE_FPA3))
> +		cvmx_fpa1_disable();
> +	/* FPA3 does not have a disable function */
> +}
> +
> +/**
> + * @INTERNAL
> + * @deprecated OBSOLETE
> + *
> + * Kept for transition assistance only
> + */
> +static inline void cvmx_fpa_global_initialize(void)
> +{
> +	cvmx_fpa_global_init_node(cvmx_get_node_num());
> +}
> +
> +/**
> + * @INTERNAL
> + *
> + * Convert FPA1 style POOL into FPA3 AURA in
> + * backward compatibility mode.
> + */
> +static inline cvmx_fpa3_gaura_t
> cvmx_fpa1_pool_to_fpa3_aura(cvmx_fpa1_pool_t pool)
> +{
> +	if ((octeon_has_feature(OCTEON_FEATURE_FPA3))) {
> +		unsigned int node = cvmx_get_node_num();
> +		cvmx_fpa3_gaura_t aura = __cvmx_fpa3_gaura(node, pool);
> +		return aura;
> +	}
> +	return CVMX_FPA3_INVALID_GAURA;
> +}
> +
> +/**
> + * Get a new block from the FPA
> + *
> + * @param pool   Pool to get the block from
> + * @return Pointer to the block or NULL on failure
> + */
> +static inline void *cvmx_fpa_alloc(u64 pool)
> +{
> +	/* FPA3 is handled differently */
> +	if ((octeon_has_feature(OCTEON_FEATURE_FPA3))) {
> +		return
> cvmx_fpa3_alloc(cvmx_fpa1_pool_to_fpa3_aura(pool));
> +	} else
> +		return cvmx_fpa1_alloc(pool);
> +}
> +
> +/**
> + * Asynchronously get a new block from the FPA
> + *
> + * The result of cvmx_fpa_async_alloc() may be retrieved using
> + * cvmx_fpa_async_alloc_finish().
> + *
> + * @param scr_addr Local scratch address to put response in.  This
> is a byte
> + *		   address but must be 8 byte aligned.
> + * @param pool      Pool to get the block from
> + */
> +static inline void cvmx_fpa_async_alloc(u64 scr_addr, u64 pool)
> +{
> +	if ((octeon_has_feature(OCTEON_FEATURE_FPA3))) {
> +		return cvmx_fpa3_async_alloc(scr_addr,
> cvmx_fpa1_pool_to_fpa3_aura(pool));
> +	} else
> +		return cvmx_fpa1_async_alloc(scr_addr, pool);
> +}
> +
> +/**
> + * Retrieve the result of cvmx_fpa_async_alloc
> + *
> + * @param scr_addr The Local scratch address.  Must be the same
> value
> + * passed to cvmx_fpa_async_alloc().
> + *
> + * @param pool Pool the block came from.  Must be the same value
> + * passed to cvmx_fpa_async_alloc.
> + *
> + * @return Pointer to the block or NULL on failure
> + */
> +static inline void *cvmx_fpa_async_alloc_finish(u64 scr_addr, u64
> pool)
> +{
> +	if ((octeon_has_feature(OCTEON_FEATURE_FPA3)))
> +		return cvmx_fpa3_async_alloc_finish(scr_addr,
> cvmx_fpa1_pool_to_fpa3_aura(pool));
> +	else
> +		return cvmx_fpa1_async_alloc_finish(scr_addr, pool);
> +}
> +
> +/**
> + * Free a block allocated with a FPA pool.
> + * Does NOT provide memory ordering in cases where the memory block
> was
> + * modified by the core.
> + *
> + * @param ptr    Block to free
> + * @param pool   Pool to put it in
> + * @param num_cache_lines
> + *               Cache lines to invalidate
> + */
> +static inline void cvmx_fpa_free_nosync(void *ptr, u64 pool, u64
> num_cache_lines)
> +{
> +	/* FPA3 is handled differently */
> +	if ((octeon_has_feature(OCTEON_FEATURE_FPA3)))
> +		cvmx_fpa3_free_nosync(ptr,
> cvmx_fpa1_pool_to_fpa3_aura(pool), num_cache_lines);
> +	else
> +		cvmx_fpa1_free_nosync(ptr, pool, num_cache_lines);
> +}
> +
> +/**
> + * Free a block allocated with a FPA pool.  Provides required memory
> + * ordering in cases where memory block was modified by core.
> + *
> + * @param ptr    Block to free
> + * @param pool   Pool to put it in
> + * @param num_cache_lines
> + *               Cache lines to invalidate
> + */
> +static inline void cvmx_fpa_free(void *ptr, u64 pool, u64
> num_cache_lines)
> +{
> +	if ((octeon_has_feature(OCTEON_FEATURE_FPA3)))
> +		cvmx_fpa3_free(ptr, cvmx_fpa1_pool_to_fpa3_aura(pool),
> num_cache_lines);
> +	else
> +		cvmx_fpa1_free(ptr, pool, num_cache_lines);
> +}
> +
> +/**
> + * Setup a FPA pool to control a new block of memory.
> + * This can only be called once per pool. Make sure proper
> + * locking enforces this.
> + *
> + * @param pool       Pool to initialize
> + * @param name       Constant character string to name this pool.
> + *                   String is not copied.
> + * @param buffer     Pointer to the block of memory to use. This
> must be
> + *                   accessible by all processors and external
> hardware.
> + * @param block_size Size for each block controlled by the FPA
> + * @param num_blocks Number of blocks
> + *
> + * @return the pool number on Success,
> + *         -1 on failure
> + */
> +int cvmx_fpa_setup_pool(int pool, const char *name, void *buffer,
> u64 block_size, u64 num_blocks);
> +
> +int cvmx_fpa_shutdown_pool(int pool);
> +
> +/**
> + * Gets the block size of buffer in specified pool
> + * @param pool	 Pool to get the block size from
> + * @return       Size of buffer in specified pool
> + */
> +unsigned int cvmx_fpa_get_block_size(int pool);
> +
> +int cvmx_fpa_is_pool_available(int pool_num);
> +u64 cvmx_fpa_get_pool_owner(int pool_num);
> +int cvmx_fpa_get_max_pools(void);
> +int cvmx_fpa_get_current_count(int pool_num);
> +int cvmx_fpa_validate_pool(int pool);
> +
> +#endif /*  __CVM_FPA_H__ */
> diff --git a/arch/mips/mach-octeon/include/mach/cvmx-fpa1.h
> b/arch/mips/mach-octeon/include/mach/cvmx-fpa1.h
> new file mode 100644
> index 000000000000..6985083a5d66
> --- /dev/null
> +++ b/arch/mips/mach-octeon/include/mach/cvmx-fpa1.h
> @@ -0,0 +1,196 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +/*
> + * Copyright (C) 2020 Marvell International Ltd.
> + *
> + * Interface to the hardware Free Pool Allocator on Octeon chips.
> + * These are the legacy models, i.e. prior to CN78XX/CN76XX.
> + */
> +
> +#ifndef __CVMX_FPA1_HW_H__
> +#define __CVMX_FPA1_HW_H__
> +
> +#include "cvmx-scratch.h"
> +#include "cvmx-fpa-defs.h"
> +#include "cvmx-fpa3.h"
> +
> +/* Legacy pool range is 0..7 and 8 on CN68XX */
> +typedef int cvmx_fpa1_pool_t;
> +
> +#define CVMX_FPA1_NUM_POOLS    8
> +#define CVMX_FPA1_INVALID_POOL ((cvmx_fpa1_pool_t)-1)
> +#define CVMX_FPA1_NAME_SIZE    16
> +
> +/**
> + * Structure describing the data format used for stores to the FPA.
> + */
> +typedef union {
> +	u64 u64;
> +	struct {
> +		u64 scraddr : 8;
> +		u64 len : 8;
> +		u64 did : 8;
> +		u64 addr : 40;
> +	} s;
> +} cvmx_fpa1_iobdma_data_t;
> +
> +/*
> + * Allocate or reserve the specified fpa pool.
> + *
> + * @param pool	  FPA pool to allocate/reserve. If -1 it
> + *                finds an empty pool to allocate.
> + * @return        Alloctaed pool number or CVMX_FPA1_POOL_INVALID
> + *                if fails to allocate the pool
> + */
> +cvmx_fpa1_pool_t cvmx_fpa1_reserve_pool(cvmx_fpa1_pool_t pool);
> +
> +/**
> + * Free the specified fpa pool.
> + * @param pool	   Pool to free
> + * @return         0 for success -1 failure
> + */
> +int cvmx_fpa1_release_pool(cvmx_fpa1_pool_t pool);
> +
> +static inline void cvmx_fpa1_free(void *ptr, cvmx_fpa1_pool_t pool,
> u64 num_cache_lines)
> +{
> +	cvmx_addr_t newptr;
> +
> +	newptr.u64 = cvmx_ptr_to_phys(ptr);
> +	newptr.sfilldidspace.didspace =
> CVMX_ADDR_DIDSPACE(CVMX_FULL_DID(CVMX_OCT_DID_FPA, pool));
> +	/* Make sure that any previous writes to memory go out before
> we free
> +	 * this buffer.  This also serves as a barrier to prevent GCC
> from
> +	 * reordering operations to after the free.
> +	 */
> +	CVMX_SYNCWS;
> +	/* value written is number of cache lines not written back */
> +	cvmx_write_io(newptr.u64, num_cache_lines);
> +}
> +
> +static inline void cvmx_fpa1_free_nosync(void *ptr, cvmx_fpa1_pool_t
> pool,
> +					 unsigned int num_cache_lines)
> +{
> +	cvmx_addr_t newptr;
> +
> +	newptr.u64 = cvmx_ptr_to_phys(ptr);
> +	newptr.sfilldidspace.didspace =
> CVMX_ADDR_DIDSPACE(CVMX_FULL_DID(CVMX_OCT_DID_FPA, pool));
> +	/* Prevent GCC from reordering around free */
> +	asm volatile("" : : : "memory");
> +	/* value written is number of cache lines not written back */
> +	cvmx_write_io(newptr.u64, num_cache_lines);
> +}
> +
> +/**
> + * Enable the FPA for use. Must be performed after any CSR
> + * configuration but before any other FPA functions.
> + */
> +static inline void cvmx_fpa1_enable(void)
> +{
> +	cvmx_fpa_ctl_status_t status;
> +
> +	status.u64 = csr_rd(CVMX_FPA_CTL_STATUS);
> +	if (status.s.enb) {
> +		/*
> +		 * CN68XXP1 should not reset the FPA (doing so may
> break
> +		 * the SSO, so we may end up enabling it more than
> once.
> +		 * Just return and don't spew messages.
> +		 */
> +		return;
> +	}
> +
> +	status.u64 = 0;
> +	status.s.enb = 1;
> +	csr_wr(CVMX_FPA_CTL_STATUS, status.u64);
> +}
> +
> +/**
> + * Reset FPA to disable. Make sure buffers from all FPA pools are
> freed
> + * before disabling FPA.
> + */
> +static inline void cvmx_fpa1_disable(void)
> +{
> +	cvmx_fpa_ctl_status_t status;
> +
> +	if (OCTEON_IS_MODEL(OCTEON_CN68XX_PASS1))
> +		return;
> +
> +	status.u64 = csr_rd(CVMX_FPA_CTL_STATUS);
> +	status.s.reset = 1;
> +	csr_wr(CVMX_FPA_CTL_STATUS, status.u64);
> +}
> +
> +static inline void *cvmx_fpa1_alloc(cvmx_fpa1_pool_t pool)
> +{
> +	u64 address;
> +
> +	for (;;) {
> +		address =
> csr_rd(CVMX_ADDR_DID(CVMX_FULL_DID(CVMX_OCT_DID_FPA, pool)));
> +		if (cvmx_likely(address)) {
> +			return cvmx_phys_to_ptr(address);
> +		} else {
> +			if (csr_rd(CVMX_FPA_QUEX_AVAILABLE(pool)) > 0)
> +				udelay(50);
> +			else
> +				return NULL;
> +		}
> +	}
> +}
> +
> +/**
> + * Asynchronously get a new block from the FPA
> + * @INTERNAL
> + *
> + * The result of cvmx_fpa_async_alloc() may be retrieved using
> + * cvmx_fpa_async_alloc_finish().
> + *
> + * @param scr_addr Local scratch address to put response in.  This
> is a byte
> + *		   address but must be 8 byte aligned.
> + * @param pool      Pool to get the block from
> + */
> +static inline void cvmx_fpa1_async_alloc(u64 scr_addr,
> cvmx_fpa1_pool_t pool)
> +{
> +	cvmx_fpa1_iobdma_data_t data;
> +
> +	/* Hardware only uses 64 bit aligned locations, so convert from
> byte
> +	 * address to 64-bit index
> +	 */
> +	data.u64 = 0ull;
> +	data.s.scraddr = scr_addr >> 3;
> +	data.s.len = 1;
> +	data.s.did = CVMX_FULL_DID(CVMX_OCT_DID_FPA, pool);
> +	data.s.addr = 0;
> +
> +	cvmx_scratch_write64(scr_addr, 0ull);
> +	CVMX_SYNCW;
> +	cvmx_send_single(data.u64);
> +}
> +
> +/**
> + * Retrieve the result of cvmx_fpa_async_alloc
> + * @INTERNAL
> + *
> + * @param scr_addr The Local scratch address.  Must be the same
> value
> + * passed to cvmx_fpa_async_alloc().
> + *
> + * @param pool Pool the block came from.  Must be the same value
> + * passed to cvmx_fpa_async_alloc.
> + *
> + * @return Pointer to the block or NULL on failure
> + */
> +static inline void *cvmx_fpa1_async_alloc_finish(u64 scr_addr,
> cvmx_fpa1_pool_t pool)
> +{
> +	u64 address;
> +
> +	CVMX_SYNCIOBDMA;
> +
> +	address = cvmx_scratch_read64(scr_addr);
> +	if (cvmx_likely(address))
> +		return cvmx_phys_to_ptr(address);
> +	else
> +		return cvmx_fpa1_alloc(pool);
> +}
> +
> +static inline u64 cvmx_fpa1_get_available(cvmx_fpa1_pool_t pool)
> +{
> +	return csr_rd(CVMX_FPA_QUEX_AVAILABLE(pool));
> +}
> +
> +#endif /* __CVMX_FPA1_HW_H__ */
> diff --git a/arch/mips/mach-octeon/include/mach/cvmx-fpa3.h
> b/arch/mips/mach-octeon/include/mach/cvmx-fpa3.h
> new file mode 100644
> index 000000000000..229982b83163
> --- /dev/null
> +++ b/arch/mips/mach-octeon/include/mach/cvmx-fpa3.h
> @@ -0,0 +1,566 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +/*
> + * Copyright (C) 2020 Marvell International Ltd.
> + *
> + * Interface to the CN78XX Free Pool Allocator, a.k.a. FPA3
> + */
> +
> +#include "cvmx-address.h"
> +#include "cvmx-fpa-defs.h"
> +#include "cvmx-scratch.h"
> +
> +#ifndef __CVMX_FPA3_H__
> +#define __CVMX_FPA3_H__
> +
> +typedef struct {
> +	unsigned res0 : 6;
> +	unsigned node : 2;
> +	unsigned res1 : 2;
> +	unsigned lpool : 6;
> +	unsigned valid_magic : 16;
> +} cvmx_fpa3_pool_t;
> +
> +typedef struct {
> +	unsigned res0 : 6;
> +	unsigned node : 2;
> +	unsigned res1 : 6;
> +	unsigned laura : 10;
> +	unsigned valid_magic : 16;
> +} cvmx_fpa3_gaura_t;
> +
> +#define CVMX_FPA3_VALID_MAGIC	0xf9a3
> +#define CVMX_FPA3_INVALID_GAURA ((cvmx_fpa3_gaura_t){ 0, 0, 0, 0, 0
> })
> +#define CVMX_FPA3_INVALID_POOL	((cvmx_fpa3_pool_t){ 0, 0, 0,
> 0, 0 })
> +
> +static inline bool __cvmx_fpa3_aura_valid(cvmx_fpa3_gaura_t aura)
> +{
> +	if (aura.valid_magic != CVMX_FPA3_VALID_MAGIC)
> +		return false;
> +	return true;
> +}
> +
> +static inline bool __cvmx_fpa3_pool_valid(cvmx_fpa3_pool_t pool)
> +{
> +	if (pool.valid_magic != CVMX_FPA3_VALID_MAGIC)
> +		return false;
> +	return true;
> +}
> +
> +static inline cvmx_fpa3_gaura_t __cvmx_fpa3_gaura(int node, int
> laura)
> +{
> +	cvmx_fpa3_gaura_t aura;
> +
> +	if (node < 0)
> +		node = cvmx_get_node_num();
> +	if (laura < 0)
> +		return CVMX_FPA3_INVALID_GAURA;
> +
> +	aura.node = node;
> +	aura.laura = laura;
> +	aura.valid_magic = CVMX_FPA3_VALID_MAGIC;
> +	return aura;
> +}
> +
> +static inline cvmx_fpa3_pool_t __cvmx_fpa3_pool(int node, int lpool)
> +{
> +	cvmx_fpa3_pool_t pool;
> +
> +	if (node < 0)
> +		node = cvmx_get_node_num();
> +	if (lpool < 0)
> +		return CVMX_FPA3_INVALID_POOL;
> +
> +	pool.node = node;
> +	pool.lpool = lpool;
> +	pool.valid_magic = CVMX_FPA3_VALID_MAGIC;
> +	return pool;
> +}
> +
> +#undef CVMX_FPA3_VALID_MAGIC
> +
> +/**
> + * Structure describing the data format used for stores to the FPA.
> + */
> +typedef union {
> +	u64 u64;
> +	struct {
> +		u64 scraddr : 8;
> +		u64 len : 8;
> +		u64 did : 8;
> +		u64 addr : 40;
> +	} s;
> +	struct {
> +		u64 scraddr : 8;
> +		u64 len : 8;
> +		u64 did : 8;
> +		u64 node : 4;
> +		u64 red : 1;
> +		u64 reserved2 : 9;
> +		u64 aura : 10;
> +		u64 reserved3 : 16;
> +	} cn78xx;
> +} cvmx_fpa3_iobdma_data_t;
> +
> +/**
> + * Struct describing load allocate operation addresses for FPA pool.
> + */
> +union cvmx_fpa3_load_data {
> +	u64 u64;
> +	struct {
> +		u64 seg : 2;
> +		u64 reserved1 : 13;
> +		u64 io : 1;
> +		u64 did : 8;
> +		u64 node : 4;
> +		u64 red : 1;
> +		u64 reserved2 : 9;
> +		u64 aura : 10;
> +		u64 reserved3 : 16;
> +	};
> +};
> +
> +typedef union cvmx_fpa3_load_data cvmx_fpa3_load_data_t;
> +
> +/**
> + * Struct describing store free operation addresses from FPA pool.
> + */
> +union cvmx_fpa3_store_addr {
> +	u64 u64;
> +	struct {
> +		u64 seg : 2;
> +		u64 reserved1 : 13;
> +		u64 io : 1;
> +		u64 did : 8;
> +		u64 node : 4;
> +		u64 reserved2 : 10;
> +		u64 aura : 10;
> +		u64 fabs : 1;
> +		u64 reserved3 : 3;
> +		u64 dwb_count : 9;
> +		u64 reserved4 : 3;
> +	};
> +};
> +
> +typedef union cvmx_fpa3_store_addr cvmx_fpa3_store_addr_t;
> +
> +enum cvmx_fpa3_pool_alignment_e {
> +	FPA_NATURAL_ALIGNMENT,
> +	FPA_OFFSET_ALIGNMENT,
> +	FPA_OPAQUE_ALIGNMENT
> +};
> +
> +#define CVMX_FPA3_AURAX_LIMIT_MAX ((1ull << 40) - 1)
> +
> +/**
> + * @INTERNAL
> + * Accessor functions to return number of POOLS in an FPA3
> + * depending on SoC model.
> + * The number is per-node for models supporting multi-node
> configurations.
> + */
> +static inline int cvmx_fpa3_num_pools(void)
> +{
> +	if (OCTEON_IS_MODEL(OCTEON_CN78XX))
> +		return 64;
> +	if (OCTEON_IS_MODEL(OCTEON_CNF75XX))
> +		return 32;
> +	if (OCTEON_IS_MODEL(OCTEON_CN73XX))
> +		return 32;
> +	printf("ERROR: %s: Unknowm model\n", __func__);
> +	return -1;
> +}
> +
> +/**
> + * @INTERNAL
> + * Accessor functions to return number of AURAS in an FPA3
> + * depending on SoC model.
> + * The number is per-node for models supporting multi-node
> configurations.
> + */
> +static inline int cvmx_fpa3_num_auras(void)
> +{
> +	if (OCTEON_IS_MODEL(OCTEON_CN78XX))
> +		return 1024;
> +	if (OCTEON_IS_MODEL(OCTEON_CNF75XX))
> +		return 512;
> +	if (OCTEON_IS_MODEL(OCTEON_CN73XX))
> +		return 512;
> +	printf("ERROR: %s: Unknowm model\n", __func__);
> +	return -1;
> +}
> +
> +/**
> + * Get the FPA3 POOL underneath FPA3 AURA, containing all its
> buffers
> + *
> + */
> +static inline cvmx_fpa3_pool_t
> cvmx_fpa3_aura_to_pool(cvmx_fpa3_gaura_t aura)
> +{
> +	cvmx_fpa3_pool_t pool;
> +	cvmx_fpa_aurax_pool_t aurax_pool;
> +
> +	aurax_pool.u64 = cvmx_read_csr_node(aura.node,
> CVMX_FPA_AURAX_POOL(aura.laura));
> +
> +	pool = __cvmx_fpa3_pool(aura.node, aurax_pool.s.pool);
> +	return pool;
> +}
> +
> +/**
> + * Get a new block from the FPA pool
> + *
> + * @param aura  - aura number
> + * @return pointer to the block or NULL on failure
> + */
> +static inline void *cvmx_fpa3_alloc(cvmx_fpa3_gaura_t aura)
> +{
> +	u64 address;
> +	cvmx_fpa3_load_data_t load_addr;
> +
> +	load_addr.u64 = 0;
> +	load_addr.seg = CVMX_MIPS_SPACE_XKPHYS;
> +	load_addr.io = 1;
> +	load_addr.did = 0x29; /* Device ID. Indicates FPA. */
> +	load_addr.node = aura.node;
> +	load_addr.red = 0; /* Perform RED on allocation.
> +				  * FIXME to use config option
> +				  */
> +	load_addr.aura = aura.laura;
> +
> +	address = cvmx_read64_uint64(load_addr.u64);
> +	if (!address)
> +		return NULL;
> +	return cvmx_phys_to_ptr(address);
> +}
> +
> +/**
> + * Asynchronously get a new block from the FPA
> + *
> + * The result of cvmx_fpa_async_alloc() may be retrieved using
> + * cvmx_fpa_async_alloc_finish().
> + *
> + * @param scr_addr Local scratch address to put response in.  This
> is a byte
> + *		   address but must be 8 byte aligned.
> + * @param aura     Global aura to get the block from
> + */
> +static inline void cvmx_fpa3_async_alloc(u64 scr_addr,
> cvmx_fpa3_gaura_t aura)
> +{
> +	cvmx_fpa3_iobdma_data_t data;
> +
> +	/* Hardware only uses 64 bit aligned locations, so convert from
> byte
> +	 * address to 64-bit index
> +	 */
> +	data.u64 = 0ull;
> +	data.cn78xx.scraddr = scr_addr >> 3;
> +	data.cn78xx.len = 1;
> +	data.cn78xx.did = 0x29;
> +	data.cn78xx.node = aura.node;
> +	data.cn78xx.aura = aura.laura;
> +	cvmx_scratch_write64(scr_addr, 0ull);
> +
> +	CVMX_SYNCW;
> +	cvmx_send_single(data.u64);
> +}
> +
> +/**
> + * Retrieve the result of cvmx_fpa3_async_alloc
> + *
> + * @param scr_addr The Local scratch address.  Must be the same
> value
> + * passed to cvmx_fpa_async_alloc().
> + *
> + * @param aura Global aura the block came from.  Must be the same
> value
> + * passed to cvmx_fpa_async_alloc.
> + *
> + * @return Pointer to the block or NULL on failure
> + */
> +static inline void *cvmx_fpa3_async_alloc_finish(u64 scr_addr,
> cvmx_fpa3_gaura_t aura)
> +{
> +	u64 address;
> +
> +	CVMX_SYNCIOBDMA;
> +
> +	address = cvmx_scratch_read64(scr_addr);
> +	if (cvmx_likely(address))
> +		return cvmx_phys_to_ptr(address);
> +	else
> +		/* Try regular alloc if async failed */
> +		return cvmx_fpa3_alloc(aura);
> +}
> +
> +/**
> + * Free a pointer back to the pool.
> + *
> + * @param aura   global aura number
> + * @param ptr    physical address of block to free.
> + * @param num_cache_lines Cache lines to invalidate
> + */
> +static inline void cvmx_fpa3_free(void *ptr, cvmx_fpa3_gaura_t aura,
> unsigned int num_cache_lines)
> +{
> +	cvmx_fpa3_store_addr_t newptr;
> +	cvmx_addr_t newdata;
> +
> +	newdata.u64 = cvmx_ptr_to_phys(ptr);
> +
> +	/* Make sure that any previous writes to memory go out before
> we free
> +	   this buffer. This also serves as a barrier to prevent GCC
> from
> +	   reordering operations to after the free. */
> +	CVMX_SYNCWS;
> +
> +	newptr.u64 = 0;
> +	newptr.seg = CVMX_MIPS_SPACE_XKPHYS;
> +	newptr.io = 1;
> +	newptr.did = 0x29; /* Device id, indicates FPA */
> +	newptr.node = aura.node;
> +	newptr.aura = aura.laura;
> +	newptr.fabs = 0; /* Free absolute. FIXME to use config option
> */
> +	newptr.dwb_count = num_cache_lines;
> +
> +	cvmx_write_io(newptr.u64, newdata.u64);
> +}
> +
> +/**
> + * Free a pointer back to the pool without flushing the write
> buffer.
> + *
> + * @param aura   global aura number
> + * @param ptr    physical address of block to free.
> + * @param num_cache_lines Cache lines to invalidate
> + */
> +static inline void cvmx_fpa3_free_nosync(void *ptr,
> cvmx_fpa3_gaura_t aura,
> +					 unsigned int num_cache_lines)
> +{
> +	cvmx_fpa3_store_addr_t newptr;
> +	cvmx_addr_t newdata;
> +
> +	newdata.u64 = cvmx_ptr_to_phys(ptr);
> +
> +	/* Prevent GCC from reordering writes to (*ptr) */
> +	asm volatile("" : : : "memory");
> +
> +	newptr.u64 = 0;
> +	newptr.seg = CVMX_MIPS_SPACE_XKPHYS;
> +	newptr.io = 1;
> +	newptr.did = 0x29; /* Device id, indicates FPA */
> +	newptr.node = aura.node;
> +	newptr.aura = aura.laura;
> +	newptr.fabs = 0; /* Free absolute. FIXME to use config option
> */
> +	newptr.dwb_count = num_cache_lines;
> +
> +	cvmx_write_io(newptr.u64, newdata.u64);
> +}
> +
> +static inline int cvmx_fpa3_pool_is_enabled(cvmx_fpa3_pool_t pool)
> +{
> +	cvmx_fpa_poolx_cfg_t pool_cfg;
> +
> +	if (!__cvmx_fpa3_pool_valid(pool))
> +		return -1;
> +
> +	pool_cfg.u64 = cvmx_read_csr_node(pool.node,
> CVMX_FPA_POOLX_CFG(pool.lpool));
> +	return pool_cfg.cn78xx.ena;
> +}
> +
> +static inline int cvmx_fpa3_config_red_params(unsigned int node, int
> qos_avg_en, int red_lvl_dly,
> +					      int avg_dly)
> +{
> +	cvmx_fpa_gen_cfg_t fpa_cfg;
> +	cvmx_fpa_red_delay_t red_delay;
> +
> +	fpa_cfg.u64 = cvmx_read_csr_node(node, CVMX_FPA_GEN_CFG);
> +	fpa_cfg.s.avg_en = qos_avg_en;
> +	fpa_cfg.s.lvl_dly = red_lvl_dly;
> +	cvmx_write_csr_node(node, CVMX_FPA_GEN_CFG, fpa_cfg.u64);
> +
> +	red_delay.u64 = cvmx_read_csr_node(node, CVMX_FPA_RED_DELAY);
> +	red_delay.s.avg_dly = avg_dly;
> +	cvmx_write_csr_node(node, CVMX_FPA_RED_DELAY, red_delay.u64);
> +	return 0;
> +}
> +
> +/**
> + * Gets the buffer size of the specified pool,
> + *
> + * @param aura Global aura number
> + * @return Returns size of the buffers in the specified pool.
> + */
> +static inline int cvmx_fpa3_get_aura_buf_size(cvmx_fpa3_gaura_t
> aura)
> +{
> +	cvmx_fpa3_pool_t pool;
> +	cvmx_fpa_poolx_cfg_t pool_cfg;
> +	int block_size;
> +
> +	pool = cvmx_fpa3_aura_to_pool(aura);
> +
> +	pool_cfg.u64 = cvmx_read_csr_node(pool.node,
> CVMX_FPA_POOLX_CFG(pool.lpool));
> +	block_size = pool_cfg.cn78xx.buf_size << 7;
> +	return block_size;
> +}
> +
> +/**
> + * Return the number of available buffers in an AURA
> + *
> + * @param aura to receive count for
> + * @return available buffer count
> + */
> +static inline long long cvmx_fpa3_get_available(cvmx_fpa3_gaura_t
> aura)
> +{
> +	cvmx_fpa3_pool_t pool;
> +	cvmx_fpa_poolx_available_t avail_reg;
> +	cvmx_fpa_aurax_cnt_t cnt_reg;
> +	cvmx_fpa_aurax_cnt_limit_t limit_reg;
> +	long long ret;
> +
> +	pool = cvmx_fpa3_aura_to_pool(aura);
> +
> +	/* Get POOL available buffer count */
> +	avail_reg.u64 = cvmx_read_csr_node(pool.node,
> CVMX_FPA_POOLX_AVAILABLE(pool.lpool));
> +
> +	/* Get AURA current available count */
> +	cnt_reg.u64 = cvmx_read_csr_node(aura.node,
> CVMX_FPA_AURAX_CNT(aura.laura));
> +	limit_reg.u64 = cvmx_read_csr_node(aura.node,
> CVMX_FPA_AURAX_CNT_LIMIT(aura.laura));
> +
> +	if (limit_reg.cn78xx.limit < cnt_reg.cn78xx.cnt)
> +		return 0;
> +
> +	/* Calculate AURA-based buffer allowance */
> +	ret = limit_reg.cn78xx.limit - cnt_reg.cn78xx.cnt;
> +
> +	/* Use POOL real buffer availability when less then allowance
> */
> +	if (ret > (long long)avail_reg.cn78xx.count)
> +		ret = avail_reg.cn78xx.count;
> +
> +	return ret;
> +}
> +
> +/**
> + * Configure the QoS parameters of an FPA3 AURA
> + *
> + * @param aura is the FPA3 AURA handle
> + * @param ena_bp enables backpressure when outstanding count exceeds
> 'bp_thresh'
> + * @param ena_red enables random early discard when outstanding
> count exceeds 'pass_thresh'
> + * @param pass_thresh is the maximum count to invoke flow control
> + * @param drop_thresh is the count threshold to begin dropping
> packets
> + * @param bp_thresh is the back-pressure threshold
> + *
> + */
> +static inline void cvmx_fpa3_setup_aura_qos(cvmx_fpa3_gaura_t aura,
> bool ena_red, u64 pass_thresh,
> +					    u64 drop_thresh, bool
> ena_bp, u64 bp_thresh)
> +{
> +	unsigned int shift = 0;
> +	u64 shift_thresh;
> +	cvmx_fpa_aurax_cnt_limit_t limit_reg;
> +	cvmx_fpa_aurax_cnt_levels_t aura_level;
> +
> +	if (!__cvmx_fpa3_aura_valid(aura))
> +		return;
> +
> +	/* Get AURAX count limit for validation */
> +	limit_reg.u64 = cvmx_read_csr_node(aura.node,
> CVMX_FPA_AURAX_CNT_LIMIT(aura.laura));
> +
> +	if (pass_thresh < 256)
> +		pass_thresh = 255;
> +
> +	if (drop_thresh <= pass_thresh || drop_thresh >
> limit_reg.cn78xx.limit)
> +		drop_thresh = limit_reg.cn78xx.limit;
> +
> +	if (bp_thresh < 256 || bp_thresh > limit_reg.cn78xx.limit)
> +		bp_thresh = limit_reg.cn78xx.limit >> 1;
> +
> +	shift_thresh = (bp_thresh > drop_thresh) ? bp_thresh :
> drop_thresh;
> +
> +	/* Calculate shift so that the largest threshold fits in 8 bits
> */
> +	for (shift = 0; shift < (1 << 6); shift++) {
> +		if (0 == ((shift_thresh >> shift) & ~0xffull))
> +			break;
> +	};
> +
> +	aura_level.u64 = cvmx_read_csr_node(aura.node,
> CVMX_FPA_AURAX_CNT_LEVELS(aura.laura));
> +	aura_level.s.pass = pass_thresh >> shift;
> +	aura_level.s.drop = drop_thresh >> shift;
> +	aura_level.s.bp = bp_thresh >> shift;
> +	aura_level.s.shift = shift;
> +	aura_level.s.red_ena = ena_red;
> +	aura_level.s.bp_ena = ena_bp;
> +	cvmx_write_csr_node(aura.node,
> CVMX_FPA_AURAX_CNT_LEVELS(aura.laura), aura_level.u64);
> +}
> +
> +cvmx_fpa3_gaura_t cvmx_fpa3_reserve_aura(int node, int
> desired_aura_num);
> +int cvmx_fpa3_release_aura(cvmx_fpa3_gaura_t aura);
> +cvmx_fpa3_pool_t cvmx_fpa3_reserve_pool(int node, int
> desired_pool_num);
> +int cvmx_fpa3_release_pool(cvmx_fpa3_pool_t pool);
> +int cvmx_fpa3_is_aura_available(int node, int aura_num);
> +int cvmx_fpa3_is_pool_available(int node, int pool_num);
> +
> +cvmx_fpa3_pool_t cvmx_fpa3_setup_fill_pool(int node, int
> desired_pool, const char *name,
> +					   unsigned int block_size,
> unsigned int num_blocks,
> +					   void *buffer);
> +
> +/**
> + * Function to attach an aura to an existing pool
> + *
> + * @param node - configure fpa on this node
> + * @param pool - configured pool to attach aura to
> + * @param desired_aura - pointer to aura to use, set to -1 to
> allocate
> + * @param name - name to register
> + * @param block_size - size of buffers to use
> + * @param num_blocks - number of blocks to allocate
> + *
> + * @return configured gaura on success, CVMX_FPA3_INVALID_GAURA on
> failure
> + */
> +cvmx_fpa3_gaura_t cvmx_fpa3_set_aura_for_pool(cvmx_fpa3_pool_t pool,
> int desired_aura,
> +					      const char *name,
> unsigned int block_size,
> +					      unsigned int num_blocks);
> +
> +/**
> + * Function to setup and initialize a pool.
> + *
> + * @param node - configure fpa on this node
> + * @param desired_aura - aura to use, -1 for dynamic allocation
> + * @param name - name to register
> + * @param block_size - size of buffers in pool
> + * @param num_blocks - max number of buffers allowed
> + */
> +cvmx_fpa3_gaura_t cvmx_fpa3_setup_aura_and_pool(int node, int
> desired_aura, const char *name,
> +						void *buffer, unsigned
> int block_size,
> +						unsigned int
> num_blocks);
> +
> +int cvmx_fpa3_shutdown_aura_and_pool(cvmx_fpa3_gaura_t aura);
> +int cvmx_fpa3_shutdown_aura(cvmx_fpa3_gaura_t aura);
> +int cvmx_fpa3_shutdown_pool(cvmx_fpa3_pool_t pool);
> +const char *cvmx_fpa3_get_pool_name(cvmx_fpa3_pool_t pool);
> +int cvmx_fpa3_get_pool_buf_size(cvmx_fpa3_pool_t pool);
> +const char *cvmx_fpa3_get_aura_name(cvmx_fpa3_gaura_t aura);
> +
> +/* FIXME: Need a different macro for stage2 of u-boot */
> +
> +static inline void cvmx_fpa3_stage2_init(int aura, int pool, u64
> stack_paddr, int stacklen,
> +					 int buffer_sz, int buf_cnt)
> +{
> +	cvmx_fpa_poolx_cfg_t pool_cfg;
> +
> +	/* Configure pool stack */
> +	cvmx_write_csr_node(0, CVMX_FPA_POOLX_STACK_BASE(pool),
> stack_paddr);
> +	cvmx_write_csr_node(0, CVMX_FPA_POOLX_STACK_ADDR(pool),
> stack_paddr);
> +	cvmx_write_csr_node(0, CVMX_FPA_POOLX_STACK_END(pool),
> stack_paddr + stacklen);
> +
> +	/* Configure pool with buffer size */
> +	pool_cfg.u64 = 0;
> +	pool_cfg.cn78xx.nat_align = 1;
> +	pool_cfg.cn78xx.buf_size = buffer_sz >> 7;
> +	pool_cfg.cn78xx.l_type = 0x2;
> +	pool_cfg.cn78xx.ena = 0;
> +	cvmx_write_csr_node(0, CVMX_FPA_POOLX_CFG(pool), pool_cfg.u64);
> +	/* Reset pool before starting */
> +	pool_cfg.cn78xx.ena = 1;
> +	cvmx_write_csr_node(0, CVMX_FPA_POOLX_CFG(pool), pool_cfg.u64);
> +
> +	cvmx_write_csr_node(0, CVMX_FPA_AURAX_CFG(aura), 0);
> +	cvmx_write_csr_node(0, CVMX_FPA_AURAX_CNT_ADD(aura), buf_cnt);
> +	cvmx_write_csr_node(0, CVMX_FPA_AURAX_POOL(aura), (u64)pool);
> +}
> +
> +static inline void cvmx_fpa3_stage2_disable(int aura, int pool)
> +{
> +	cvmx_write_csr_node(0, CVMX_FPA_AURAX_POOL(aura), 0);
> +	cvmx_write_csr_node(0, CVMX_FPA_POOLX_CFG(pool), 0);
> +	cvmx_write_csr_node(0, CVMX_FPA_POOLX_STACK_BASE(pool), 0);
> +	cvmx_write_csr_node(0, CVMX_FPA_POOLX_STACK_ADDR(pool), 0);
> +	cvmx_write_csr_node(0, CVMX_FPA_POOLX_STACK_END(pool), 0);
> +}
> +
> +#endif /* __CVMX_FPA3_H__ */
> diff --git a/arch/mips/mach-octeon/include/mach/cvmx-global-
> resources.h b/arch/mips/mach-octeon/include/mach/cvmx-global-
> resources.h
> new file mode 100644
> index 000000000000..28c32ddbe17a
> --- /dev/null
> +++ b/arch/mips/mach-octeon/include/mach/cvmx-global-resources.h
> @@ -0,0 +1,213 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +/*
> + * Copyright (C) 2020 Marvell International Ltd.
> + */
> +
> +#ifndef _CVMX_GLOBAL_RESOURCES_T_
> +#define _CVMX_GLOBAL_RESOURCES_T_
> +
> +#define CVMX_GLOBAL_RESOURCES_DATA_NAME "cvmx-global-resources"
> +
> +/*In macros below abbreviation GR stands for global resources. */
> +#define
> CVMX_GR_TAG_INVALID                                                  
>                       \
> +	cvmx_get_gr_tag('i', 'n', 'v', 'a', 'l', 'i', 'd', '.', '.',
> '.', '.', '.', '.', '.', '.', \
> +			'.')
> +/*Tag for pko que table range. */
> +#define
> CVMX_GR_TAG_PKO_QUEUES                                               
>                       \
> +	cvmx_get_gr_tag('c', 'v', 'm', '_', 'p', 'k', 'o', '_', 'q',
> 'u', 'e', 'u', 's', '.', '.', \
> +			'.')
> +/*Tag for a pko internal ports range */
> +#define
> CVMX_GR_TAG_PKO_IPORTS                                               
>                       \
> +	cvmx_get_gr_tag('c', 'v', 'm', '_', 'p', 'k', 'o', '_', 'i',
> 'p', 'o', 'r', 't', '.', '.', \
> +			'.')
> +#define
> CVMX_GR_TAG_FPA                                                      
>                       \
> +	cvmx_get_gr_tag('c', 'v', 'm', '_', 'f', 'p', 'a', '.', '.',
> '.', '.', '.', '.', '.', '.', \
> +			'.')
> +#define
> CVMX_GR_TAG_FAU                                                      
>                       \
> +	cvmx_get_gr_tag('c', 'v', 'm', '_', 'f', 'a', 'u', '.', '.',
> '.', '.', '.', '.', '.', '.', \
> +			'.')
> +#define
> CVMX_GR_TAG_SSO_GRP(n)                                               
>                       \
> +	cvmx_get_gr_tag('c', 'v', 'm', '_', 's', 's', 'o', '_', '0',
> (n) + '0', '.', '.', '.',     \
> +			'.', '.', '.');
> +#define
> CVMX_GR_TAG_TIM(n)                                                   
>                       \
> +	cvmx_get_gr_tag('c', 'v', 'm', '_', 't', 'i', 'm', '_', (n) +
> '0', '.', '.', '.', '.',     \
> +			'.', '.', '.')
> +#define
> CVMX_GR_TAG_CLUSTERS(x)                                              
>                       \
> +	cvmx_get_gr_tag('c', 'v', 'm', '_', 'c', 'l', 'u', 's', 't',
> 'e', 'r', '_', (x + '0'),     \
> +			'.', '.', '.')
> +#define
> CVMX_GR_TAG_CLUSTER_GRP(x)                                           
>                       \
> +	cvmx_get_gr_tag('c', 'v', 'm', '_', 'c', 'l', 'g', 'r', 'p',
> '_', (x + '0'), '.', '.',     \
> +			'.', '.', '.')
> +#define
> CVMX_GR_TAG_STYLE(x)                                                 
>                       \
> +	cvmx_get_gr_tag('c', 'v', 'm', '_', 's', 't', 'y', 'l', 'e',
> '_', (x + '0'), '.', '.',     \
> +			'.', '.', '.')
> +#define
> CVMX_GR_TAG_QPG_ENTRY(x)                                             
>                       \
> +	cvmx_get_gr_tag('c', 'v', 'm', '_', 'q', 'p', 'g', 'e', 't',
> '_', (x + '0'), '.', '.',     \
> +			'.', '.', '.')
> +#define
> CVMX_GR_TAG_BPID(x)                                                  
>                       \
> +	cvmx_get_gr_tag('c', 'v', 'm', '_', 'b', 'p', 'i', 'd', 's',
> '_', (x + '0'), '.', '.',     \
> +			'.', '.', '.')
> +#define
> CVMX_GR_TAG_MTAG_IDX(x)                                              
>                       \
> +	cvmx_get_gr_tag('c', 'v', 'm', '_', 'm', 't', 'a', 'g', 'x',
> '_', (x + '0'), '.', '.',     \
> +			'.', '.', '.')
> +#define CVMX_GR_TAG_PCAM(x, y,
> z)                                                                  \
> +	cvmx_get_gr_tag('c', 'v', 'm', '_', 'p', 'c', 'a', 'm', '_', (x
> + '0'), (y + '0'),         \
> +			(z + '0'), '.', '.', '.', '.')
> +
> +#define
> CVMX_GR_TAG_CIU3_IDT(_n)                                             
>                       \
> +	cvmx_get_gr_tag('c', 'v', 'm', '_', 'c', 'i', 'u', '3', '_',
> ((_n) + '0'), '_', 'i', 'd',  \
> +			't', '.', '.')
> +
> +/* Allocation of the 512 SW INTSTs (in the  12 bit SW INTSN space)
> */
> +#define
> CVMX_GR_TAG_CIU3_SWINTSN(_n)                                         
>                       \
> +	cvmx_get_gr_tag('c', 'v', 'm', '_', 'c', 'i', 'u', '3', '_',
> ((_n) + '0'), '_', 's', 'w',  \
> +			'i', 's', 'n')
> +
> +#define TAG_INIT_PART(A, B, C, D, E, F, G,
> H)                                                      \
> +	((((u64)(A) & 0xff) << 56) | (((u64)(B) & 0xff) << 48) |
> (((u64)(C) & 0xff) << 40) |             \
> +	 (((u64)(D) & 0xff) << 32) | (((u64)(E) & 0xff) << 24) |
> (((u64)(F) & 0xff) << 16) |             \
> +	 (((u64)(G) & 0xff) << 8) | (((u64)(H) & 0xff)))
> +
> +struct global_resource_tag {
> +	u64 lo;
> +	u64 hi;
> +};
> +
> +enum cvmx_resource_err { CVMX_RESOURCE_ALLOC_FAILED = -1,
> CVMX_RESOURCE_ALREADY_RESERVED = -2 };
> +
> +/*
> + * @INTERNAL
> + * Creates a tag from the specified characters.
> + */
> +static inline struct global_resource_tag cvmx_get_gr_tag(char a,
> char b, char c, char d, char e,
> +							 char f, char
> g, char h, char i, char j,
> +							 char k, char
> l, char m, char n, char o,
> +							 char p)
> +{
> +	struct global_resource_tag tag;
> +
> +	tag.lo = TAG_INIT_PART(a, b, c, d, e, f, g, h);
> +	tag.hi = TAG_INIT_PART(i, j, k, l, m, n, o, p);
> +	return tag;
> +}
> +
> +static inline int cvmx_gr_same_tag(struct global_resource_tag gr1,
> struct global_resource_tag gr2)
> +{
> +	return (gr1.hi == gr2.hi) && (gr1.lo == gr2.lo);
> +}
> +
> +/*
> + * @INTERNAL
> + * Creates a global resource range that can hold the specified
> number of
> + * elements
> + * @param tag is the tag of the range. The taga is created using the
> method
> + * cvmx_get_gr_tag()
> + * @param nelements is the number of elements to be held in the
> resource range.
> + */
> +int cvmx_create_global_resource_range(struct global_resource_tag
> tag, int nelements);
> +
> +/*
> + * @INTERNAL
> + * Allocate nelements in the global resource range with the
> specified tag. It
> + * is assumed that prior
> + * to calling this the global resource range has already been
> created using
> + * cvmx_create_global_resource_range().
> + * @param tag is the tag of the global resource range.
> + * @param nelements is the number of elements to be allocated.
> + * @param owner is a 64 bit number that identifes the owner of this
> range.
> + * @aligment specifes the required alignment of the returned base
> number.
> + * @return returns the base of the allocated range. -1 return value
> indicates
> + * failure.
> + */
> +int cvmx_allocate_global_resource_range(struct global_resource_tag
> tag, u64 owner, int nelements,
> +					int alignment);
> +
> +/*
> + * @INTERNAL
> + * Allocate nelements in the global resource range with the
> specified tag.
> + * The elements allocated need not be contiguous. It is assumed that
> prior to
> + * calling this the global resource range has already
> + * been created using cvmx_create_global_resource_range().
> + * @param tag is the tag of the global resource range.
> + * @param nelements is the number of elements to be allocated.
> + * @param owner is a 64 bit number that identifes the owner of the
> allocated
> + * elements.
> + * @param allocated_elements returns indexs of the allocated
> entries.
> + * @return returns 0 on success and -1 on failure.
> + */
> +int cvmx_resource_alloc_many(struct global_resource_tag tag, u64
> owner, int nelements,
> +			     int allocated_elements[]);
> +int cvmx_resource_alloc_reverse(struct global_resource_tag, u64
> owner);
> +/*
> + * @INTERNAL
> + * Reserve nelements starting from base in the global resource range
> with the
> + * specified tag.
> + * It is assumed that prior to calling this the global resource
> range has
> + * already been created using cvmx_create_global_resource_range().
> + * @param tag is the tag of the global resource range.
> + * @param nelements is the number of elements to be allocated.
> + * @param owner is a 64 bit number that identifes the owner of this
> range.
> + * @base specifies the base start of nelements.
> + * @return returns the base of the allocated range. -1 return value
> indicates
> + * failure.
> + */
> +int cvmx_reserve_global_resource_range(struct global_resource_tag
> tag, u64 owner, int base,
> +				       int nelements);
> +/*
> + * @INTERNAL
> + * Free nelements starting at base in the global resource range with
> the
> + * specified tag.
> + * @param tag is the tag of the global resource range.
> + * @param base is the base number
> + * @param nelements is the number of elements that are to be freed.
> + * @return returns 0 if successful and -1 on failure.
> + */
> +int cvmx_free_global_resource_range_with_base(struct
> global_resource_tag tag, int base,
> +					      int nelements);
> +
> +/*
> + * @INTERNAL
> + * Free nelements with the bases specified in bases[] with the
> + * specified tag.
> + * @param tag is the tag of the global resource range.
> + * @param bases is an array containing the bases to be freed.
> + * @param nelements is the number of elements that are to be freed.
> + * @return returns 0 if successful and -1 on failure.
> + */
> +int cvmx_free_global_resource_range_multiple(struct
> global_resource_tag tag, int bases[],
> +					     int nelements);
> +/*
> + * @INTERNAL
> + * Free elements from the specified owner in the global resource
> range with the
> + * specified tag.
> + * @param tag is the tag of the global resource range.
> + * @param owner is the owner of resources that are to be freed.
> + * @return returns 0 if successful and -1 on failure.
> + */
> +int cvmx_free_global_resource_range_with_owner(struct
> global_resource_tag tag, int owner);
> +
> +/*
> + * @INTERNAL
> + * Frees all the global resources that have been created.
> + * For use only from the bootloader, when it shutdown and boots up
> the
> + * application or kernel.
> + */
> +int free_global_resources(void);
> +
> +u64 cvmx_get_global_resource_owner(struct global_resource_tag tag,
> int base);
> +/*
> + * @INTERNAL
> + * Shows the global resource range with the specified tag. Use
> mainly for debug.
> + */
> +void cvmx_show_global_resource_range(struct global_resource_tag
> tag);
> +
> +/*
> + * @INTERNAL
> + * Shows all the global resources. Used mainly for debug.
> + */
> +void cvmx_global_resources_show(void);
> +
> +u64 cvmx_allocate_app_id(void);
> +u64 cvmx_get_app_id(void);
> +
> +#endif
> diff --git a/arch/mips/mach-octeon/include/mach/cvmx-gmx.h
> b/arch/mips/mach-octeon/include/mach/cvmx-gmx.h
> new file mode 100644
> index 000000000000..2df7da102a0f
> --- /dev/null
> +++ b/arch/mips/mach-octeon/include/mach/cvmx-gmx.h
> @@ -0,0 +1,16 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +/*
> + * Copyright (C) 2020 Marvell International Ltd.
> + *
> + * Interface to the GMX hardware.
> + */
> +
> +#ifndef __CVMX_GMX_H__
> +#define __CVMX_GMX_H__
> +
> +/* CSR typedefs have been moved to cvmx-gmx-defs.h */
> +
> +int cvmx_gmx_set_backpressure_override(u32 interface, u32
> port_mask);
> +int cvmx_agl_set_backpressure_override(u32 interface, u32
> port_mask);
> +
> +#endif
> diff --git a/arch/mips/mach-octeon/include/mach/cvmx-hwfau.h
> b/arch/mips/mach-octeon/include/mach/cvmx-hwfau.h
> new file mode 100644
> index 000000000000..59772190aa3b
> --- /dev/null
> +++ b/arch/mips/mach-octeon/include/mach/cvmx-hwfau.h
> @@ -0,0 +1,606 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +/*
> + * Copyright (C) 2020 Marvell International Ltd.
> + *
> + * Interface to the hardware Fetch and Add Unit.
> + */
> +
> +/**
> + * @file
> + *
> + * Interface to the hardware Fetch and Add Unit.
> + *
> + */
> +
> +#ifndef __CVMX_HWFAU_H__
> +#define __CVMX_HWFAU_H__
> +
> +typedef int cvmx_fau_reg64_t;
> +typedef int cvmx_fau_reg32_t;
> +typedef int cvmx_fau_reg16_t;
> +typedef int cvmx_fau_reg8_t;
> +
> +#define CVMX_FAU_REG_ANY -1
> +
> +/*
> + * Octeon Fetch and Add Unit (FAU)
> + */
> +
> +#define CVMX_FAU_LOAD_IO_ADDRESS cvmx_build_io_address(0x1e, 0)
> +#define CVMX_FAU_BITS_SCRADDR	 63, 56
> +#define CVMX_FAU_BITS_LEN	 55, 48
> +#define CVMX_FAU_BITS_INEVAL	 35, 14
> +#define CVMX_FAU_BITS_TAGWAIT	 13, 13
> +#define CVMX_FAU_BITS_NOADD	 13, 13
> +#define CVMX_FAU_BITS_SIZE	 12, 11
> +#define CVMX_FAU_BITS_REGISTER	 10, 0
> +
> +#define CVMX_FAU_MAX_REGISTERS_8 (2048)
> +
> +typedef enum {
> +	CVMX_FAU_OP_SIZE_8 = 0,
> +	CVMX_FAU_OP_SIZE_16 = 1,
> +	CVMX_FAU_OP_SIZE_32 = 2,
> +	CVMX_FAU_OP_SIZE_64 = 3
> +} cvmx_fau_op_size_t;
> +
> +/**
> + * Tagwait return definition. If a timeout occurs, the error
> + * bit will be set. Otherwise the value of the register before
> + * the update will be returned.
> + */
> +typedef struct {
> +	u64 error : 1;
> +	s64 value : 63;
> +} cvmx_fau_tagwait64_t;
> +
> +/**
> + * Tagwait return definition. If a timeout occurs, the error
> + * bit will be set. Otherwise the value of the register before
> + * the update will be returned.
> + */
> +typedef struct {
> +	u64 error : 1;
> +	s32 value : 31;
> +} cvmx_fau_tagwait32_t;
> +
> +/**
> + * Tagwait return definition. If a timeout occurs, the error
> + * bit will be set. Otherwise the value of the register before
> + * the update will be returned.
> + */
> +typedef struct {
> +	u64 error : 1;
> +	s16 value : 15;
> +} cvmx_fau_tagwait16_t;
> +
> +/**
> + * Tagwait return definition. If a timeout occurs, the error
> + * bit will be set. Otherwise the value of the register before
> + * the update will be returned.
> + */
> +typedef struct {
> +	u64 error : 1;
> +	int8_t value : 7;
> +} cvmx_fau_tagwait8_t;
> +
> +/**
> + * Asynchronous tagwait return definition. If a timeout occurs,
> + * the error bit will be set. Otherwise the value of the
> + * register before the update will be returned.
> + */
> +typedef union {
> +	u64 u64;
> +	struct {
> +		u64 invalid : 1;
> +		u64 data : 63; /* unpredictable if invalid is set */
> +	} s;
> +} cvmx_fau_async_tagwait_result_t;
> +
> +#define SWIZZLE_8  0
> +#define SWIZZLE_16 0
> +#define SWIZZLE_32 0
> +
> +/**
> + * @INTERNAL
> + * Builds a store I/O address for writing to the FAU
> + *
> + * @param noadd  0 = Store value is atomically added to the current
> value
> + *               1 = Store value is atomically written over the
> current value
> + * @param reg    FAU atomic register to access. 0 <= reg < 2048.
> + *               - Step by 2 for 16 bit access.
> + *               - Step by 4 for 32 bit access.
> + *               - Step by 8 for 64 bit access.
> + * @return Address to store for atomic update
> + */
> +static inline u64 __cvmx_hwfau_store_address(u64 noadd, u64 reg)
> +{
> +	return (CVMX_ADD_IO_SEG(CVMX_FAU_LOAD_IO_ADDRESS) |
> +		cvmx_build_bits(CVMX_FAU_BITS_NOADD, noadd) |
> +		cvmx_build_bits(CVMX_FAU_BITS_REGISTER, reg));
> +}
> +
> +/**
> + * @INTERNAL
> + * Builds a I/O address for accessing the FAU
> + *
> + * @param tagwait Should the atomic add wait for the current tag
> switch
> + *                operation to complete.
> + *                - 0 = Don't wait
> + *                - 1 = Wait for tag switch to complete
> + * @param reg     FAU atomic register to access. 0 <= reg < 2048.
> + *                - Step by 2 for 16 bit access.
> + *                - Step by 4 for 32 bit access.
> + *                - Step by 8 for 64 bit access.
> + * @param value   Signed value to add.
> + *                Note: When performing 32 and 64 bit access, only
> the low
> + *                22 bits are available.
> + * @return Address to read from for atomic update
> + */
> +static inline u64 __cvmx_hwfau_atomic_address(u64 tagwait, u64 reg,
> s64 value)
> +{
> +	return (CVMX_ADD_IO_SEG(CVMX_FAU_LOAD_IO_ADDRESS) |
> +		cvmx_build_bits(CVMX_FAU_BITS_INEVAL, value) |
> +		cvmx_build_bits(CVMX_FAU_BITS_TAGWAIT, tagwait) |
> +		cvmx_build_bits(CVMX_FAU_BITS_REGISTER, reg));
> +}
> +
> +/**
> + * Perform an atomic 64 bit add
> + *
> + * @param reg     FAU atomic register to access. 0 <= reg < 2048.
> + *                - Step by 8 for 64 bit access.
> + * @param value   Signed value to add.
> + *                Note: Only the low 22 bits are available.
> + * @return Value of the register before the update
> + */
> +static inline s64 cvmx_hwfau_fetch_and_add64(cvmx_fau_reg64_t reg,
> s64 value)
> +{
> +	return cvmx_read64_int64(__cvmx_hwfau_atomic_address(0, reg,
> value));
> +}
> +
> +/**
> + * Perform an atomic 32 bit add
> + *
> + * @param reg     FAU atomic register to access. 0 <= reg < 2048.
> + *                - Step by 4 for 32 bit access.
> + * @param value   Signed value to add.
> + *                Note: Only the low 22 bits are available.
> + * @return Value of the register before the update
> + */
> +static inline s32 cvmx_hwfau_fetch_and_add32(cvmx_fau_reg32_t reg,
> s32 value)
> +{
> +	reg ^= SWIZZLE_32;
> +	return cvmx_read64_int32(__cvmx_hwfau_atomic_address(0, reg,
> value));
> +}
> +
> +/**
> + * Perform an atomic 16 bit add
> + *
> + * @param reg     FAU atomic register to access. 0 <= reg < 2048.
> + *                - Step by 2 for 16 bit access.
> + * @param value   Signed value to add.
> + * @return Value of the register before the update
> + */
> +static inline s16 cvmx_hwfau_fetch_and_add16(cvmx_fau_reg16_t reg,
> s16 value)
> +{
> +	reg ^= SWIZZLE_16;
> +	return cvmx_read64_int16(__cvmx_hwfau_atomic_address(0, reg,
> value));
> +}
> +
> +/**
> + * Perform an atomic 8 bit add
> + *
> + * @param reg     FAU atomic register to access. 0 <= reg < 2048.
> + * @param value   Signed value to add.
> + * @return Value of the register before the update
> + */
> +static inline int8_t cvmx_hwfau_fetch_and_add8(cvmx_fau_reg8_t reg,
> int8_t value)
> +{
> +	reg ^= SWIZZLE_8;
> +	return cvmx_read64_int8(__cvmx_hwfau_atomic_address(0, reg,
> value));
> +}
> +
> +/**
> + * Perform an atomic 64 bit add after the current tag switch
> + * completes
> + *
> + * @param reg    FAU atomic register to access. 0 <= reg < 2048.
> + *               - Step by 8 for 64 bit access.
> + * @param value  Signed value to add.
> + *               Note: Only the low 22 bits are available.
> + * @return If a timeout occurs, the error bit will be set. Otherwise
> + *         the value of the register before the update will be
> + *         returned
> + */
> +static inline cvmx_fau_tagwait64_t
> cvmx_hwfau_tagwait_fetch_and_add64(cvmx_fau_reg64_t reg,
> +								      s
> 64 value)
> +{
> +	union {
> +		u64 i64;
> +		cvmx_fau_tagwait64_t t;
> +	} result;
> +	result.i64 = cvmx_read64_int64(__cvmx_hwfau_atomic_address(1,
> reg, value));
> +	return result.t;
> +}
> +
> +/**
> + * Perform an atomic 32 bit add after the current tag switch
> + * completes
> + *
> + * @param reg    FAU atomic register to access. 0 <= reg < 2048.
> + *               - Step by 4 for 32 bit access.
> + * @param value  Signed value to add.
> + *               Note: Only the low 22 bits are available.
> + * @return If a timeout occurs, the error bit will be set. Otherwise
> + *         the value of the register before the update will be
> + *         returned
> + */
> +static inline cvmx_fau_tagwait32_t
> cvmx_hwfau_tagwait_fetch_and_add32(cvmx_fau_reg32_t reg,
> +								      s
> 32 value)
> +{
> +	union {
> +		u64 i32;
> +		cvmx_fau_tagwait32_t t;
> +	} result;
> +	reg ^= SWIZZLE_32;
> +	result.i32 = cvmx_read64_int32(__cvmx_hwfau_atomic_address(1,
> reg, value));
> +	return result.t;
> +}
> +
> +/**
> + * Perform an atomic 16 bit add after the current tag switch
> + * completes
> + *
> + * @param reg    FAU atomic register to access. 0 <= reg < 2048.
> + *               - Step by 2 for 16 bit access.
> + * @param value  Signed value to add.
> + * @return If a timeout occurs, the error bit will be set. Otherwise
> + *         the value of the register before the update will be
> + *         returned
> + */
> +static inline cvmx_fau_tagwait16_t
> cvmx_hwfau_tagwait_fetch_and_add16(cvmx_fau_reg16_t reg,
> +								      s
> 16 value)
> +{
> +	union {
> +		u64 i16;
> +		cvmx_fau_tagwait16_t t;
> +	} result;
> +	reg ^= SWIZZLE_16;
> +	result.i16 = cvmx_read64_int16(__cvmx_hwfau_atomic_address(1,
> reg, value));
> +	return result.t;
> +}
> +
> +/**
> + * Perform an atomic 8 bit add after the current tag switch
> + * completes
> + *
> + * @param reg    FAU atomic register to access. 0 <= reg < 2048.
> + * @param value  Signed value to add.
> + * @return If a timeout occurs, the error bit will be set. Otherwise
> + *         the value of the register before the update will be
> + *         returned
> + */
> +static inline cvmx_fau_tagwait8_t
> cvmx_hwfau_tagwait_fetch_and_add8(cvmx_fau_reg8_t reg,
> +								    int
> 8_t value)
> +{
> +	union {
> +		u64 i8;
> +		cvmx_fau_tagwait8_t t;
> +	} result;
> +	reg ^= SWIZZLE_8;
> +	result.i8 = cvmx_read64_int8(__cvmx_hwfau_atomic_address(1,
> reg, value));
> +	return result.t;
> +}
> +
> +/**
> + * @INTERNAL
> + * Builds I/O data for async operations
> + *
> + * @param scraddr Scratch pad byte address to write to.  Must be 8
> byte aligned
> + * @param value   Signed value to add.
> + *                Note: When performing 32 and 64 bit access, only
> the low
> + *                22 bits are available.
> + * @param tagwait Should the atomic add wait for the current tag
> switch
> + *                operation to complete.
> + *                - 0 = Don't wait
> + *                - 1 = Wait for tag switch to complete
> + * @param size    The size of the operation:
> + *                - CVMX_FAU_OP_SIZE_8  (0) = 8 bits
> + *                - CVMX_FAU_OP_SIZE_16 (1) = 16 bits
> + *                - CVMX_FAU_OP_SIZE_32 (2) = 32 bits
> + *                - CVMX_FAU_OP_SIZE_64 (3) = 64 bits
> + * @param reg     FAU atomic register to access. 0 <= reg < 2048.
> + *                - Step by 2 for 16 bit access.
> + *                - Step by 4 for 32 bit access.
> + *                - Step by 8 for 64 bit access.
> + * @return Data to write using cvmx_send_single
> + */
> +static inline u64 __cvmx_fau_iobdma_data(u64 scraddr, s64 value, u64
> tagwait,
> +					 cvmx_fau_op_size_t size, u64
> reg)
> +{
> +	return (CVMX_FAU_LOAD_IO_ADDRESS |
> cvmx_build_bits(CVMX_FAU_BITS_SCRADDR, scraddr >> 3) |
> +		cvmx_build_bits(CVMX_FAU_BITS_LEN, 1) |
> +		cvmx_build_bits(CVMX_FAU_BITS_INEVAL, value) |
> +		cvmx_build_bits(CVMX_FAU_BITS_TAGWAIT, tagwait) |
> +		cvmx_build_bits(CVMX_FAU_BITS_SIZE, size) |
> +		cvmx_build_bits(CVMX_FAU_BITS_REGISTER, reg));
> +}
> +
> +/**
> + * Perform an async atomic 64 bit add. The old value is
> + * placed in the scratch memory at byte address scraddr.
> + *
> + * @param scraddr Scratch memory byte address to put response in.
> + *                Must be 8 byte aligned.
> + * @param reg     FAU atomic register to access. 0 <= reg < 2048.
> + *                - Step by 8 for 64 bit access.
> + * @param value   Signed value to add.
> + *                Note: Only the low 22 bits are available.
> + * @return Placed in the scratch pad register
> + */
> +static inline void cvmx_hwfau_async_fetch_and_add64(u64 scraddr,
> cvmx_fau_reg64_t reg, s64 value)
> +{
> +	cvmx_send_single(__cvmx_fau_iobdma_data(scraddr, value, 0,
> CVMX_FAU_OP_SIZE_64, reg));
> +}
> +
> +/**
> + * Perform an async atomic 32 bit add. The old value is
> + * placed in the scratch memory at byte address scraddr.
> + *
> + * @param scraddr Scratch memory byte address to put response in.
> + *                Must be 8 byte aligned.
> + * @param reg     FAU atomic register to access. 0 <= reg < 2048.
> + *                - Step by 4 for 32 bit access.
> + * @param value   Signed value to add.
> + *                Note: Only the low 22 bits are available.
> + * @return Placed in the scratch pad register
> + */
> +static inline void cvmx_hwfau_async_fetch_and_add32(u64 scraddr,
> cvmx_fau_reg32_t reg, s32 value)
> +{
> +	cvmx_send_single(__cvmx_fau_iobdma_data(scraddr, value, 0,
> CVMX_FAU_OP_SIZE_32, reg));
> +}
> +
> +/**
> + * Perform an async atomic 16 bit add. The old value is
> + * placed in the scratch memory at byte address scraddr.
> + *
> + * @param scraddr Scratch memory byte address to put response in.
> + *                Must be 8 byte aligned.
> + * @param reg     FAU atomic register to access. 0 <= reg < 2048.
> + *                - Step by 2 for 16 bit access.
> + * @param value   Signed value to add.
> + * @return Placed in the scratch pad register
> + */
> +static inline void cvmx_hwfau_async_fetch_and_add16(u64 scraddr,
> cvmx_fau_reg16_t reg, s16 value)
> +{
> +	cvmx_send_single(__cvmx_fau_iobdma_data(scraddr, value, 0,
> CVMX_FAU_OP_SIZE_16, reg));
> +}
> +
> +/**
> + * Perform an async atomic 8 bit add. The old value is
> + * placed in the scratch memory at byte address scraddr.
> + *
> + * @param scraddr Scratch memory byte address to put response in.
> + *                Must be 8 byte aligned.
> + * @param reg     FAU atomic register to access. 0 <= reg < 2048.
> + * @param value   Signed value to add.
> + * @return Placed in the scratch pad register
> + */
> +static inline void cvmx_hwfau_async_fetch_and_add8(u64 scraddr,
> cvmx_fau_reg8_t reg, int8_t value)
> +{
> +	cvmx_send_single(__cvmx_fau_iobdma_data(scraddr, value, 0,
> CVMX_FAU_OP_SIZE_8, reg));
> +}
> +
> +/**
> + * Perform an async atomic 64 bit add after the current tag
> + * switch completes.
> + *
> + * @param scraddr Scratch memory byte address to put response in.
> + *                Must be 8 byte aligned.
> + *                If a timeout occurs, the error bit (63) will be
> set. Otherwise
> + *                the value of the register before the update will
> be
> + *                returned
> + * @param reg     FAU atomic register to access. 0 <= reg < 2048.
> + *                - Step by 8 for 64 bit access.
> + * @param value   Signed value to add.
> + *                Note: Only the low 22 bits are available.
> + * @return Placed in the scratch pad register
> + */
> +static inline void cvmx_hwfau_async_tagwait_fetch_and_add64(u64
> scraddr, cvmx_fau_reg64_t reg,
> +							    s64 value)
> +{
> +	cvmx_send_single(__cvmx_fau_iobdma_data(scraddr, value, 1,
> CVMX_FAU_OP_SIZE_64, reg));
> +}
> +
> +/**
> + * Perform an async atomic 32 bit add after the current tag
> + * switch completes.
> + *
> + * @param scraddr Scratch memory byte address to put response in.
> + *                Must be 8 byte aligned.
> + *                If a timeout occurs, the error bit (63) will be
> set. Otherwise
> + *                the value of the register before the update will
> be
> + *                returned
> + * @param reg     FAU atomic register to access. 0 <= reg < 2048.
> + *                - Step by 4 for 32 bit access.
> + * @param value   Signed value to add.
> + *                Note: Only the low 22 bits are available.
> + * @return Placed in the scratch pad register
> + */
> +static inline void cvmx_hwfau_async_tagwait_fetch_and_add32(u64
> scraddr, cvmx_fau_reg32_t reg,
> +							    s32 value)
> +{
> +	cvmx_send_single(__cvmx_fau_iobdma_data(scraddr, value, 1,
> CVMX_FAU_OP_SIZE_32, reg));
> +}
> +
> +/**
> + * Perform an async atomic 16 bit add after the current tag
> + * switch completes.
> + *
> + * @param scraddr Scratch memory byte address to put response in.
> + *                Must be 8 byte aligned.
> + *                If a timeout occurs, the error bit (63) will be
> set. Otherwise
> + *                the value of the register before the update will
> be
> + *                returned
> + * @param reg     FAU atomic register to access. 0 <= reg < 2048.
> + *                - Step by 2 for 16 bit access.
> + * @param value   Signed value to add.
> + * @return Placed in the scratch pad register
> + */
> +static inline void cvmx_hwfau_async_tagwait_fetch_and_add16(u64
> scraddr, cvmx_fau_reg16_t reg,
> +							    s16 value)
> +{
> +	cvmx_send_single(__cvmx_fau_iobdma_data(scraddr, value, 1,
> CVMX_FAU_OP_SIZE_16, reg));
> +}
> +
> +/**
> + * Perform an async atomic 8 bit add after the current tag
> + * switch completes.
> + *
> + * @param scraddr Scratch memory byte address to put response in.
> + *                Must be 8 byte aligned.
> + *                If a timeout occurs, the error bit (63) will be
> set. Otherwise
> + *                the value of the register before the update will
> be
> + *                returned
> + * @param reg     FAU atomic register to access. 0 <= reg < 2048.
> + * @param value   Signed value to add.
> + * @return Placed in the scratch pad register
> + */
> +static inline void cvmx_hwfau_async_tagwait_fetch_and_add8(u64
> scraddr, cvmx_fau_reg8_t reg,
> +							   int8_t
> value)
> +{
> +	cvmx_send_single(__cvmx_fau_iobdma_data(scraddr, value, 1,
> CVMX_FAU_OP_SIZE_8, reg));
> +}
> +
> +/**
> + * Perform an atomic 64 bit add
> + *
> + * @param reg     FAU atomic register to access. 0 <= reg < 2048.
> + *                - Step by 8 for 64 bit access.
> + * @param value   Signed value to add.
> + */
> +static inline void cvmx_hwfau_atomic_add64(cvmx_fau_reg64_t reg, s64
> value)
> +{
> +	cvmx_write64_int64(__cvmx_hwfau_store_address(0, reg), value);
> +}
> +
> +/**
> + * Perform an atomic 32 bit add
> + *
> + * @param reg     FAU atomic register to access. 0 <= reg < 2048.
> + *                - Step by 4 for 32 bit access.
> + * @param value   Signed value to add.
> + */
> +static inline void cvmx_hwfau_atomic_add32(cvmx_fau_reg32_t reg, s32
> value)
> +{
> +	reg ^= SWIZZLE_32;
> +	cvmx_write64_int32(__cvmx_hwfau_store_address(0, reg), value);
> +}
> +
> +/**
> + * Perform an atomic 16 bit add
> + *
> + * @param reg     FAU atomic register to access. 0 <= reg < 2048.
> + *                - Step by 2 for 16 bit access.
> + * @param value   Signed value to add.
> + */
> +static inline void cvmx_hwfau_atomic_add16(cvmx_fau_reg16_t reg, s16
> value)
> +{
> +	reg ^= SWIZZLE_16;
> +	cvmx_write64_int16(__cvmx_hwfau_store_address(0, reg), value);
> +}
> +
> +/**
> + * Perform an atomic 8 bit add
> + *
> + * @param reg     FAU atomic register to access. 0 <= reg < 2048.
> + * @param value   Signed value to add.
> + */
> +static inline void cvmx_hwfau_atomic_add8(cvmx_fau_reg8_t reg,
> int8_t value)
> +{
> +	reg ^= SWIZZLE_8;
> +	cvmx_write64_int8(__cvmx_hwfau_store_address(0, reg), value);
> +}
> +
> +/**
> + * Perform an atomic 64 bit write
> + *
> + * @param reg     FAU atomic register to access. 0 <= reg < 2048.
> + *                - Step by 8 for 64 bit access.
> + * @param value   Signed value to write.
> + */
> +static inline void cvmx_hwfau_atomic_write64(cvmx_fau_reg64_t reg,
> s64 value)
> +{
> +	cvmx_write64_int64(__cvmx_hwfau_store_address(1, reg), value);
> +}
> +
> +/**
> + * Perform an atomic 32 bit write
> + *
> + * @param reg     FAU atomic register to access. 0 <= reg < 2048.
> + *                - Step by 4 for 32 bit access.
> + * @param value   Signed value to write.
> + */
> +static inline void cvmx_hwfau_atomic_write32(cvmx_fau_reg32_t reg,
> s32 value)
> +{
> +	reg ^= SWIZZLE_32;
> +	cvmx_write64_int32(__cvmx_hwfau_store_address(1, reg), value);
> +}
> +
> +/**
> + * Perform an atomic 16 bit write
> + *
> + * @param reg     FAU atomic register to access. 0 <= reg < 2048.
> + *                - Step by 2 for 16 bit access.
> + * @param value   Signed value to write.
> + */
> +static inline void cvmx_hwfau_atomic_write16(cvmx_fau_reg16_t reg,
> s16 value)
> +{
> +	reg ^= SWIZZLE_16;
> +	cvmx_write64_int16(__cvmx_hwfau_store_address(1, reg), value);
> +}
> +
> +/**
> + * Perform an atomic 8 bit write
> + *
> + * @param reg     FAU atomic register to access. 0 <= reg < 2048.
> + * @param value   Signed value to write.
> + */
> +static inline void cvmx_hwfau_atomic_write8(cvmx_fau_reg8_t reg,
> int8_t value)
> +{
> +	reg ^= SWIZZLE_8;
> +	cvmx_write64_int8(__cvmx_hwfau_store_address(1, reg), value);
> +}
> +
> +/** Allocates 64bit FAU register.
> + *  @return value is the base address of allocated FAU register
> + */
> +int cvmx_fau64_alloc(int reserve);
> +
> +/** Allocates 32bit FAU register.
> + *  @return value is the base address of allocated FAU register
> + */
> +int cvmx_fau32_alloc(int reserve);
> +
> +/** Allocates 16bit FAU register.
> + *  @return value is the base address of allocated FAU register
> + */
> +int cvmx_fau16_alloc(int reserve);
> +
> +/** Allocates 8bit FAU register.
> + *  @return value is the base address of allocated FAU register
> + */
> +int cvmx_fau8_alloc(int reserve);
> +
> +/** Frees the specified FAU register.
> + *  @param address Base address of register to release.
> + *  @return 0 on success; -1 on failure
> + */
> +int cvmx_fau_free(int address);
> +
> +/** Display the fau registers array
> + */
> +void cvmx_fau_show(void);
> +
> +#endif /* __CVMX_HWFAU_H__ */
> diff --git a/arch/mips/mach-octeon/include/mach/cvmx-hwpko.h
> b/arch/mips/mach-octeon/include/mach/cvmx-hwpko.h
> new file mode 100644
> index 000000000000..459c19bbc0f1
> --- /dev/null
> +++ b/arch/mips/mach-octeon/include/mach/cvmx-hwpko.h
> @@ -0,0 +1,570 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +/*
> + * Copyright (C) 2020 Marvell International Ltd.
> + *
> + * Interface to the hardware Packet Output unit.
> + *
> + * Starting with SDK 1.7.0, the PKO output functions now support
> + * two types of locking. CVMX_PKO_LOCK_ATOMIC_TAG continues to
> + * function similarly to previous SDKs by using POW atomic tags
> + * to preserve ordering and exclusivity. As a new option, you
> + * can now pass CVMX_PKO_LOCK_CMD_QUEUE which uses a ll/sc
> + * memory based locking instead. This locking has the advantage
> + * of not affecting the tag state but doesn't preserve packet
> + * ordering. CVMX_PKO_LOCK_CMD_QUEUE is appropriate in most
> + * generic code while CVMX_PKO_LOCK_CMD_QUEUE should be used
> + * with hand tuned fast path code.
> + *
> + * Some of other SDK differences visible to the command command
> + * queuing:
> + * - PKO indexes are no longer stored in the FAU. A large
> + *   percentage of the FAU register block used to be tied up
> + *   maintaining PKO queue pointers. These are now stored in a
> + *   global named block.
> + * - The PKO <b>use_locking</b> parameter can now have a global
> + *   effect. Since all application use the same named block,
> + *   queue locking correctly applies across all operating
> + *   systems when using CVMX_PKO_LOCK_CMD_QUEUE.
> + * - PKO 3 word commands are now supported. Use
> + *   cvmx_pko_send_packet_finish3().
> + */
> +
> +#ifndef __CVMX_HWPKO_H__
> +#define __CVMX_HWPKO_H__
> +
> +#include "cvmx-hwfau.h"
> +#include "cvmx-fpa.h"
> +#include "cvmx-pow.h"
> +#include "cvmx-cmd-queue.h"
> +#include "cvmx-helper.h"
> +#include "cvmx-helper-util.h"
> +#include "cvmx-helper-cfg.h"
> +
> +/* Adjust the command buffer size by 1 word so that in the case of
> using only
> +** two word PKO commands no command words stradle buffers.  The
> useful values
> +** for this are 0 and 1. */
> +#define CVMX_PKO_COMMAND_BUFFER_SIZE_ADJUST (1)
> +
> +#define CVMX_PKO_MAX_OUTPUT_QUEUES_STATIC 256
> +#define
> CVMX_PKO_MAX_OUTPUT_QUEUES                                           
>                       \
> +	((OCTEON_IS_OCTEON2() || OCTEON_IS_MODEL(OCTEON_CN70XX)) ? 256
> : 128)
> +#define
> CVMX_PKO_NUM_OUTPUT_PORTS                                            
>                       \
> +	((OCTEON_IS_MODEL(OCTEON_CN63XX)) ? 44 :
> (OCTEON_IS_MODEL(OCTEON_CN66XX) ? 48 : 40))
> +#define CVMX_PKO_MEM_QUEUE_PTRS_ILLEGAL_PID 63
> +#define CVMX_PKO_QUEUE_STATIC_PRIORITY	    9
> +#define CVMX_PKO_ILLEGAL_QUEUE		    0xFFFF
> +#define CVMX_PKO_MAX_QUEUE_DEPTH	    0
> +
> +typedef enum {
> +	CVMX_PKO_SUCCESS,
> +	CVMX_PKO_INVALID_PORT,
> +	CVMX_PKO_INVALID_QUEUE,
> +	CVMX_PKO_INVALID_PRIORITY,
> +	CVMX_PKO_NO_MEMORY,
> +	CVMX_PKO_PORT_ALREADY_SETUP,
> +	CVMX_PKO_CMD_QUEUE_INIT_ERROR
> +} cvmx_pko_return_value_t;
> +
> +/**
> + * This enumeration represents the differnet locking modes supported
> by PKO.
> + */
> +typedef enum {
> +	CVMX_PKO_LOCK_NONE = 0,
> +	CVMX_PKO_LOCK_ATOMIC_TAG = 1,
> +	CVMX_PKO_LOCK_CMD_QUEUE = 2,
> +} cvmx_pko_lock_t;
> +
> +typedef struct cvmx_pko_port_status {
> +	u32 packets;
> +	u64 octets;
> +	u64 doorbell;
> +} cvmx_pko_port_status_t;
> +
> +/**
> + * This structure defines the address to use on a packet enqueue
> + */
> +typedef union {
> +	u64 u64;
> +	struct {
> +		cvmx_mips_space_t mem_space : 2;
> +		u64 reserved : 13;
> +		u64 is_io : 1;
> +		u64 did : 8;
> +		u64 reserved2 : 4;
> +		u64 reserved3 : 15;
> +		u64 port : 9;
> +		u64 queue : 9;
> +		u64 reserved4 : 3;
> +	} s;
> +} cvmx_pko_doorbell_address_t;
> +
> +/**
> + * Structure of the first packet output command word.
> + */
> +typedef union {
> +	u64 u64;
> +	struct {
> +		cvmx_fau_op_size_t size1 : 2;
> +		cvmx_fau_op_size_t size0 : 2;
> +		u64 subone1 : 1;
> +		u64 reg1 : 11;
> +		u64 subone0 : 1;
> +		u64 reg0 : 11;
> +		u64 le : 1;
> +		u64 n2 : 1;
> +		u64 wqp : 1;
> +		u64 rsp : 1;
> +		u64 gather : 1;
> +		u64 ipoffp1 : 7;
> +		u64 ignore_i : 1;
> +		u64 dontfree : 1;
> +		u64 segs : 6;
> +		u64 total_bytes : 16;
> +	} s;
> +} cvmx_pko_command_word0_t;
> +
> +/**
> + * Call before any other calls to initialize the packet
> + * output system.
> + */
> +
> +void cvmx_pko_hw_init(u8 pool, unsigned int bufsize);
> +
> +/**
> + * Enables the packet output hardware. It must already be
> + * configured.
> + */
> +void cvmx_pko_enable(void);
> +
> +/**
> + * Disables the packet output. Does not affect any configuration.
> + */
> +void cvmx_pko_disable(void);
> +
> +/**
> + * Shutdown and free resources required by packet output.
> + */
> +
> +void cvmx_pko_shutdown(void);
> +
> +/**
> + * Configure a output port and the associated queues for use.
> + *
> + * @param port       Port to configure.
> + * @param base_queue First queue number to associate with this port.
> + * @param num_queues Number of queues t oassociate with this port
> + * @param priority   Array of priority levels for each queue. Values
> are
> + *                   allowed to be 1-8. A value of 8 get 8 times the
> traffic
> + *                   of a value of 1. There must be num_queues
> elements in the
> + *                   array.
> + */
> +cvmx_pko_return_value_t cvmx_pko_config_port(int port, int
> base_queue, int num_queues,
> +					     const u8 priority[]);
> +
> +/**
> + * Ring the packet output doorbell. This tells the packet
> + * output hardware that "len" command words have been added
> + * to its pending list.  This command includes the required
> + * CVMX_SYNCWS before the doorbell ring.
> + *
> + * WARNING: This function may have to look up the proper PKO port in
> + * the IPD port to PKO port map, and is thus slower than calling
> + * cvmx_pko_doorbell_pkoid() directly if the PKO port identifier is
> + * known.
> + *
> + * @param ipd_port   The IPD port corresponding the to pko port the
> packet is for
> + * @param queue  Queue the packet is for
> + * @param len    Length of the command in 64 bit words
> + */
> +static inline void cvmx_pko_doorbell(u64 ipd_port, u64 queue, u64
> len)
> +{
> +	cvmx_pko_doorbell_address_t ptr;
> +	u64 pko_port;
> +
> +	pko_port = ipd_port;
> +	if (octeon_has_feature(OCTEON_FEATURE_PKND))
> +		pko_port = cvmx_helper_cfg_ipd2pko_port_base(ipd_port);
> +
> +	ptr.u64 = 0;
> +	ptr.s.mem_space = CVMX_IO_SEG;
> +	ptr.s.did = CVMX_OCT_DID_PKT_SEND;
> +	ptr.s.is_io = 1;
> +	ptr.s.port = pko_port;
> +	ptr.s.queue = queue;
> +	/* Need to make sure output queue data is in DRAM before
> doorbell write */
> +	CVMX_SYNCWS;
> +	cvmx_write_io(ptr.u64, len);
> +}
> +
> +/**
> + * Prepare to send a packet.  This may initiate a tag switch to
> + * get exclusive access to the output queue structure, and
> + * performs other prep work for the packet send operation.
> + *
> + * cvmx_pko_send_packet_finish() MUST be called after this function
> is called,
> + * and must be called with the same port/queue/use_locking
> arguments.
> + *
> + * The use_locking parameter allows the caller to use three
> + * possible locking modes.
> + * - CVMX_PKO_LOCK_NONE
> + *      - PKO doesn't do any locking. It is the responsibility
> + *          of the application to make sure that no other core
> + *          is accessing the same queue at the same time.
> + * - CVMX_PKO_LOCK_ATOMIC_TAG
> + *      - PKO performs an atomic tagswitch to insure exclusive
> + *          access to the output queue. This will maintain
> + *          packet ordering on output.
> + * - CVMX_PKO_LOCK_CMD_QUEUE
> + *      - PKO uses the common command queue locks to insure
> + *          exclusive access to the output queue. This is a
> + *          memory based ll/sc. This is the most portable
> + *          locking mechanism.
> + *
> + * NOTE: If atomic locking is used, the POW entry CANNOT be
> + * descheduled, as it does not contain a valid WQE pointer.
> + *
> + * @param port   Port to send it on, this can be either IPD port or
> PKO
> + *		 port.
> + * @param queue  Queue to use
> + * @param use_locking
> + *               CVMX_PKO_LOCK_NONE, CVMX_PKO_LOCK_ATOMIC_TAG, or
> CVMX_PKO_LOCK_CMD_QUEUE
> + */
> +static inline void cvmx_pko_send_packet_prepare(u64 port
> __attribute__((unused)), u64 queue,
> +						cvmx_pko_lock_t
> use_locking)
> +{
> +	if (use_locking == CVMX_PKO_LOCK_ATOMIC_TAG) {
> +		/*
> +		 * Must do a full switch here to handle all cases.  We
> use a
> +		 * fake WQE pointer, as the POW does not access this
> memory.
> +		 * The WQE pointer and group are only used if this work
> is
> +		 * descheduled, which is not supported by the
> +		 *
> cvmx_pko_send_packet_prepare/cvmx_pko_send_packet_finish
> +		 * combination. Note that this is a special case in
> which these
> +		 * fake values can be used - this is not a general
> technique.
> +		 */
> +		u32 tag = CVMX_TAG_SW_BITS_INTERNAL <<
> CVMX_TAG_SW_SHIFT |
> +			  CVMX_TAG_SUBGROUP_PKO <<
> CVMX_TAG_SUBGROUP_SHIFT |
> +			  (CVMX_TAG_SUBGROUP_MASK & queue);
> +		cvmx_pow_tag_sw_full((cvmx_wqe_t
> *)cvmx_phys_to_ptr(0x80), tag,
> +				     CVMX_POW_TAG_TYPE_ATOMIC, 0);
> +	}
> +}
> +
> +#define cvmx_pko_send_packet_prepare_pkoid
> cvmx_pko_send_packet_prepare
> +
> +/**
> + * Complete packet output. cvmx_pko_send_packet_prepare() must be
> called exactly once before this,
> + * and the same parameters must be passed to both
> cvmx_pko_send_packet_prepare() and
> + * cvmx_pko_send_packet_finish().
> + *
> + * WARNING: This function may have to look up the proper PKO port in
> + * the IPD port to PKO port map, and is thus slower than calling
> + * cvmx_pko_send_packet_finish_pkoid() directly if the PKO port
> + * identifier is known.
> + *
> + * @param ipd_port   The IPD port corresponding the to pko port the
> packet is for
> + * @param queue  Queue to use
> + * @param pko_command
> + *               PKO HW command word
> + * @param packet Packet to send
> + * @param use_locking
> + *               CVMX_PKO_LOCK_NONE, CVMX_PKO_LOCK_ATOMIC_TAG, or
> CVMX_PKO_LOCK_CMD_QUEUE
> + *
> + * @return returns CVMX_PKO_SUCCESS on success, or error code on
> failure of output
> + */
> +static inline cvmx_pko_return_value_t
> +cvmx_hwpko_send_packet_finish(u64 ipd_port, u64 queue,
> cvmx_pko_command_word0_t pko_command,
> +			      cvmx_buf_ptr_t packet, cvmx_pko_lock_t
> use_locking)
> +{
> +	cvmx_cmd_queue_result_t result;
> +
> +	if (use_locking == CVMX_PKO_LOCK_ATOMIC_TAG)
> +		cvmx_pow_tag_sw_wait();
> +
> +	result = cvmx_cmd_queue_write2(CVMX_CMD_QUEUE_PKO(queue),
> +				       (use_locking ==
> CVMX_PKO_LOCK_CMD_QUEUE), pko_command.u64,
> +				       packet.u64);
> +	if (cvmx_likely(result == CVMX_CMD_QUEUE_SUCCESS)) {
> +		cvmx_pko_doorbell(ipd_port, queue, 2);
> +		return CVMX_PKO_SUCCESS;
> +	} else if ((result == CVMX_CMD_QUEUE_NO_MEMORY) || (result ==
> CVMX_CMD_QUEUE_FULL)) {
> +		return CVMX_PKO_NO_MEMORY;
> +	} else {
> +		return CVMX_PKO_INVALID_QUEUE;
> +	}
> +}
> +
> +/**
> + * Complete packet output. cvmx_pko_send_packet_prepare() must be
> called exactly once before this,
> + * and the same parameters must be passed to both
> cvmx_pko_send_packet_prepare() and
> + * cvmx_pko_send_packet_finish().
> + *
> + * WARNING: This function may have to look up the proper PKO port in
> + * the IPD port to PKO port map, and is thus slower than calling
> + * cvmx_pko_send_packet_finish3_pkoid() directly if the PKO port
> + * identifier is known.
> + *
> + * @param ipd_port   The IPD port corresponding the to pko port the
> packet is for
> + * @param queue  Queue to use
> + * @param pko_command
> + *               PKO HW command word
> + * @param packet Packet to send
> + * @param addr   Plysical address of a work queue entry or physical
> address to zero on complete.
> + * @param use_locking
> + *               CVMX_PKO_LOCK_NONE, CVMX_PKO_LOCK_ATOMIC_TAG, or
> CVMX_PKO_LOCK_CMD_QUEUE
> + *
> + * @return returns CVMX_PKO_SUCCESS on success, or error code on
> failure of output
> + */
> +static inline cvmx_pko_return_value_t
> +cvmx_hwpko_send_packet_finish3(u64 ipd_port, u64 queue,
> cvmx_pko_command_word0_t pko_command,
> +			       cvmx_buf_ptr_t packet, u64 addr,
> cvmx_pko_lock_t use_locking)
> +{
> +	cvmx_cmd_queue_result_t result;
> +
> +	if (use_locking == CVMX_PKO_LOCK_ATOMIC_TAG)
> +		cvmx_pow_tag_sw_wait();
> +
> +	result = cvmx_cmd_queue_write3(CVMX_CMD_QUEUE_PKO(queue),
> +				       (use_locking ==
> CVMX_PKO_LOCK_CMD_QUEUE), pko_command.u64,
> +				       packet.u64, addr);
> +	if (cvmx_likely(result == CVMX_CMD_QUEUE_SUCCESS)) {
> +		cvmx_pko_doorbell(ipd_port, queue, 3);
> +		return CVMX_PKO_SUCCESS;
> +	} else if ((result == CVMX_CMD_QUEUE_NO_MEMORY) || (result ==
> CVMX_CMD_QUEUE_FULL)) {
> +		return CVMX_PKO_NO_MEMORY;
> +	} else {
> +		return CVMX_PKO_INVALID_QUEUE;
> +	}
> +}
> +
> +/**
> + * Get the first pko_port for the (interface, index)
> + *
> + * @param interface
> + * @param index
> + */
> +int cvmx_pko_get_base_pko_port(int interface, int index);
> +
> +/**
> + * Get the number of pko_ports for the (interface, index)
> + *
> + * @param interface
> + * @param index
> + */
> +int cvmx_pko_get_num_pko_ports(int interface, int index);
> +
> +/**
> + * For a given port number, return the base pko output queue
> + * for the port.
> + *
> + * @param port   IPD port number
> + * @return Base output queue
> + */
> +int cvmx_pko_get_base_queue(int port);
> +
> +/**
> + * For a given port number, return the number of pko output queues.
> + *
> + * @param port   IPD port number
> + * @return Number of output queues
> + */
> +int cvmx_pko_get_num_queues(int port);
> +
> +/**
> + * Sets the internal FPA pool data structure for PKO comamnd queue.
> + * @param pool	fpa pool number yo use
> + * @param buffer_size	buffer size of pool
> + * @param buffer_count	number of buufers to allocate to pool
> + *
> + * @note the caller is responsable for setting up the pool with
> + * an appropriate buffer size and sufficient buffer count.
> + */
> +void cvmx_pko_set_cmd_que_pool_config(s64 pool, u64 buffer_size, u64
> buffer_count);
> +
> +/**
> + * Get the status counters for a port.
> + *
> + * @param ipd_port Port number (ipd_port) to get statistics for.
> + * @param clear    Set to 1 to clear the counters after they are
> read
> + * @param status   Where to put the results.
> + *
> + * Note:
> + *     - Only the doorbell for the base queue of the ipd_port is
> + *       collected.
> + *     - Retrieving the stats involves writing the index through
> + *       CVMX_PKO_REG_READ_IDX and reading the stat CSRs, in that
> + *       order. It is not MP-safe and caller should guarantee
> + *       atomicity.
> + */
> +void cvmx_pko_get_port_status(u64 ipd_port, u64 clear,
> cvmx_pko_port_status_t *status);
> +
> +/**
> + * Rate limit a PKO port to a max packets/sec. This function is only
> + * supported on CN57XX, CN56XX, CN55XX, and CN54XX.
> + *
> + * @param port      Port to rate limit
> + * @param packets_s Maximum packet/sec
> + * @param burst     Maximum number of packets to burst in a row
> before rate
> + *                  limiting cuts in.
> + *
> + * @return Zero on success, negative on failure
> + */
> +int cvmx_pko_rate_limit_packets(int port, int packets_s, int burst);
> +
> +/**
> + * Rate limit a PKO port to a max bits/sec. This function is only
> + * supported on CN57XX, CN56XX, CN55XX, and CN54XX.
> + *
> + * @param port   Port to rate limit
> + * @param bits_s PKO rate limit in bits/sec
> + * @param burst  Maximum number of bits to burst before rate
> + *               limiting cuts in.
> + *
> + * @return Zero on success, negative on failure
> + */
> +int cvmx_pko_rate_limit_bits(int port, u64 bits_s, int burst);
> +
> +/**
> + * @INTERNAL
> + *
> + * Retrieve the PKO pipe number for a port
> + *
> + * @param interface
> + * @param index
> + *
> + * @return negative on error.
> + *
> + * This applies only to the non-loopback interfaces.
> + *
> + */
> +int __cvmx_pko_get_pipe(int interface, int index);
> +
> +/**
> + * For a given PKO port number, return the base output queue
> + * for the port.
> + *
> + * @param pko_port   PKO port number
> + * @return           Base output queue
> + */
> +int cvmx_pko_get_base_queue_pkoid(int pko_port);
> +
> +/**
> + * For a given PKO port number, return the number of output queues
> + * for the port.
> + *
> + * @param pko_port	PKO port number
> + * @return		the number of output queues
> + */
> +int cvmx_pko_get_num_queues_pkoid(int pko_port);
> +
> +/**
> + * Ring the packet output doorbell. This tells the packet
> + * output hardware that "len" command words have been added
> + * to its pending list.  This command includes the required
> + * CVMX_SYNCWS before the doorbell ring.
> + *
> + * @param pko_port   Port the packet is for
> + * @param queue  Queue the packet is for
> + * @param len    Length of the command in 64 bit words
> + */
> +static inline void cvmx_pko_doorbell_pkoid(u64 pko_port, u64 queue,
> u64 len)
> +{
> +	cvmx_pko_doorbell_address_t ptr;
> +
> +	ptr.u64 = 0;
> +	ptr.s.mem_space = CVMX_IO_SEG;
> +	ptr.s.did = CVMX_OCT_DID_PKT_SEND;
> +	ptr.s.is_io = 1;
> +	ptr.s.port = pko_port;
> +	ptr.s.queue = queue;
> +	/* Need to make sure output queue data is in DRAM before
> doorbell write */
> +	CVMX_SYNCWS;
> +	cvmx_write_io(ptr.u64, len);
> +}
> +
> +/**
> + * Complete packet output. cvmx_pko_send_packet_prepare() must be
> called exactly once before this,
> + * and the same parameters must be passed to both
> cvmx_pko_send_packet_prepare() and
> + * cvmx_pko_send_packet_finish_pkoid().
> + *
> + * @param pko_port   Port to send it on
> + * @param queue  Queue to use
> + * @param pko_command
> + *               PKO HW command word
> + * @param packet Packet to send
> + * @param use_locking
> + *               CVMX_PKO_LOCK_NONE, CVMX_PKO_LOCK_ATOMIC_TAG, or
> CVMX_PKO_LOCK_CMD_QUEUE
> + *
> + * @return returns CVMX_PKO_SUCCESS on success, or error code on
> failure of output
> + */
> +static inline cvmx_pko_return_value_t
> +cvmx_hwpko_send_packet_finish_pkoid(int pko_port, u64 queue,
> cvmx_pko_command_word0_t pko_command,
> +				    cvmx_buf_ptr_t packet,
> cvmx_pko_lock_t use_locking)
> +{
> +	cvmx_cmd_queue_result_t result;
> +
> +	if (use_locking == CVMX_PKO_LOCK_ATOMIC_TAG)
> +		cvmx_pow_tag_sw_wait();
> +
> +	result = cvmx_cmd_queue_write2(CVMX_CMD_QUEUE_PKO(queue),
> +				       (use_locking ==
> CVMX_PKO_LOCK_CMD_QUEUE), pko_command.u64,
> +				       packet.u64);
> +	if (cvmx_likely(result == CVMX_CMD_QUEUE_SUCCESS)) {
> +		cvmx_pko_doorbell_pkoid(pko_port, queue, 2);
> +		return CVMX_PKO_SUCCESS;
> +	} else if ((result == CVMX_CMD_QUEUE_NO_MEMORY) || (result ==
> CVMX_CMD_QUEUE_FULL)) {
> +		return CVMX_PKO_NO_MEMORY;
> +	} else {
> +		return CVMX_PKO_INVALID_QUEUE;
> +	}
> +}
> +
> +/**
> + * Complete packet output. cvmx_pko_send_packet_prepare() must be
> called exactly once before this,
> + * and the same parameters must be passed to both
> cvmx_pko_send_packet_prepare() and
> + * cvmx_pko_send_packet_finish_pkoid().
> + *
> + * @param pko_port   The PKO port the packet is for
> + * @param queue  Queue to use
> + * @param pko_command
> + *               PKO HW command word
> + * @param packet Packet to send
> + * @param addr   Plysical address of a work queue entry or physical
> address to zero on complete.
> + * @param use_locking
> + *               CVMX_PKO_LOCK_NONE, CVMX_PKO_LOCK_ATOMIC_TAG, or
> CVMX_PKO_LOCK_CMD_QUEUE
> + *
> + * @return returns CVMX_PKO_SUCCESS on success, or error code on
> failure of output
> + */
> +static inline cvmx_pko_return_value_t
> +cvmx_hwpko_send_packet_finish3_pkoid(u64 pko_port, u64 queue,
> cvmx_pko_command_word0_t pko_command,
> +				     cvmx_buf_ptr_t packet, u64 addr,
> cvmx_pko_lock_t use_locking)
> +{
> +	cvmx_cmd_queue_result_t result;
> +
> +	if (use_locking == CVMX_PKO_LOCK_ATOMIC_TAG)
> +		cvmx_pow_tag_sw_wait();
> +
> +	result = cvmx_cmd_queue_write3(CVMX_CMD_QUEUE_PKO(queue),
> +				       (use_locking ==
> CVMX_PKO_LOCK_CMD_QUEUE), pko_command.u64,
> +				       packet.u64, addr);
> +	if (cvmx_likely(result == CVMX_CMD_QUEUE_SUCCESS)) {
> +		cvmx_pko_doorbell_pkoid(pko_port, queue, 3);
> +		return CVMX_PKO_SUCCESS;
> +	} else if ((result == CVMX_CMD_QUEUE_NO_MEMORY) || (result ==
> CVMX_CMD_QUEUE_FULL)) {
> +		return CVMX_PKO_NO_MEMORY;
> +	} else {
> +		return CVMX_PKO_INVALID_QUEUE;
> +	}
> +}
> +
> +/*
> + * Obtain the number of PKO commands pending in a queue
> + *
> + * @param queue is the queue identifier to be queried
> + * @return the number of commands pending transmission or -1 on
> error
> + */
> +int cvmx_pko_queue_pend_count(cvmx_cmd_queue_id_t queue);
> +
> +void cvmx_pko_set_cmd_queue_pool_buffer_count(u64 buffer_count);
> +
> +#endif /* __CVMX_HWPKO_H__ */
> diff --git a/arch/mips/mach-octeon/include/mach/cvmx-ilk.h
> b/arch/mips/mach-octeon/include/mach/cvmx-ilk.h
> new file mode 100644
> index 000000000000..727298352c28
> --- /dev/null
> +++ b/arch/mips/mach-octeon/include/mach/cvmx-ilk.h
> @@ -0,0 +1,154 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +/*
> + * Copyright (C) 2020 Marvell International Ltd.
> + *
> + * This file contains defines for the ILK interface
> + */
> +
> +#ifndef __CVMX_ILK_H__
> +#define __CVMX_ILK_H__
> +
> +/* CSR typedefs have been moved to cvmx-ilk-defs.h */
> +
> +/*
> + * Note: this macro must match the first ilk port in the
> ipd_port_map_68xx[]
> + * and ipd_port_map_78xx[] arrays.
> + */
> +static inline int CVMX_ILK_GBL_BASE(void)
> +{
> +	if (OCTEON_IS_MODEL(OCTEON_CN68XX))
> +		return 5;
> +	if (OCTEON_IS_MODEL(OCTEON_CN78XX))
> +		return 6;
> +	return -1;
> +}
> +
> +static inline int CVMX_ILK_QLM_BASE(void)
> +{
> +	if (OCTEON_IS_MODEL(OCTEON_CN68XX))
> +		return 1;
> +	if (OCTEON_IS_MODEL(OCTEON_CN78XX))
> +		return 4;
> +	return -1;
> +}
> +
> +typedef struct {
> +	int intf_en : 1;
> +	int la_mode : 1;
> +	int reserved : 14; /* unused */
> +	int lane_speed : 16;
> +	/* add more here */
> +} cvmx_ilk_intf_t;
> +
> +#define CVMX_NUM_ILK_INTF 2
> +static inline int CVMX_ILK_MAX_LANES(void)
> +{
> +	if (OCTEON_IS_MODEL(OCTEON_CN68XX))
> +		return 8;
> +	if (OCTEON_IS_MODEL(OCTEON_CN78XX))
> +		return 16;
> +	return -1;
> +}
> +
> +extern unsigned short
> cvmx_ilk_lane_mask[CVMX_MAX_NODES][CVMX_NUM_ILK_INTF];
> +
> +typedef struct {
> +	unsigned int pipe;
> +	unsigned int chan;
> +} cvmx_ilk_pipe_chan_t;
> +
> +#define CVMX_ILK_MAX_PIPES 45
> +/* Max number of channels allowed */
> +#define CVMX_ILK_MAX_CHANS 256
> +
> +extern int cvmx_ilk_chans[CVMX_MAX_NODES][CVMX_NUM_ILK_INTF];
> +
> +typedef struct {
> +	unsigned int chan;
> +	unsigned int pknd;
> +} cvmx_ilk_chan_pknd_t;
> +
> +#define CVMX_ILK_MAX_PKNDS 16 /* must be <45 */
> +
> +typedef struct {
> +	int *chan_list; /* for discrete channels. or, must be null */
> +	unsigned int num_chans;
> +
> +	unsigned int chan_start; /* for continuous channels */
> +	unsigned int chan_end;
> +	unsigned int chan_step;
> +
> +	unsigned int clr_on_rd;
> +} cvmx_ilk_stats_ctrl_t;
> +
> +#define CVMX_ILK_MAX_CAL      288
> +#define CVMX_ILK_MAX_CAL_IDX  (CVMX_ILK_MAX_CAL / 8)
> +#define CVMX_ILK_TX_MIN_CAL   1
> +#define CVMX_ILK_RX_MIN_CAL   1
> +#define CVMX_ILK_CAL_GRP_SZ   8
> +#define CVMX_ILK_PIPE_BPID_SZ 7
> +#define CVMX_ILK_ENT_CTRL_SZ  2
> +#define CVMX_ILK_RX_FIFO_WM   0x200
> +
> +typedef enum { PIPE_BPID = 0, LINK, XOFF, XON }
> cvmx_ilk_cal_ent_ctrl_t;
> +
> +typedef struct {
> +	unsigned char pipe_bpid;
> +	cvmx_ilk_cal_ent_ctrl_t ent_ctrl;
> +} cvmx_ilk_cal_entry_t;
> +
> +typedef enum { CVMX_ILK_LPBK_DISA = 0, CVMX_ILK_LPBK_ENA }
> cvmx_ilk_lpbk_ena_t;
> +
> +typedef enum { CVMX_ILK_LPBK_INT = 0, CVMX_ILK_LPBK_EXT }
> cvmx_ilk_lpbk_mode_t;
> +
> +/**
> + * This header is placed in front of all received ILK look-aside
> mode packets
> + */
> +typedef union {
> +	u64 u64;
> +
> +	struct {
> +		u32 reserved_63_57 : 7;	  /* bits 63...57 */
> +		u32 nsp_cmd : 5;	  /* bits 56...52 */
> +		u32 nsp_flags : 4;	  /* bits 51...48 */
> +		u32 nsp_grp_id_upper : 6; /* bits 47...42 */
> +		u32 reserved_41_40 : 2;	  /* bits 41...40 */
> +		/* Protocol type, 1 for LA mode packet */
> +		u32 la_mode : 1;	  /* bit  39      */
> +		u32 nsp_grp_id_lower : 2; /* bits 38...37 */
> +		u32 nsp_xid_upper : 4;	  /* bits 36...33 */
> +		/* ILK channel number, 0 or 1 */
> +		u32 ilk_channel : 1;   /* bit  32      */
> +		u32 nsp_xid_lower : 8; /* bits 31...24 */
> +		/* Unpredictable, may be any value */
> +		u32 reserved_23_0 : 24; /* bits 23...0  */
> +	} s;
> +} cvmx_ilk_la_nsp_compact_hdr_t;
> +
> +typedef struct cvmx_ilk_LA_mode_struct {
> +	int ilk_LA_mode;
> +	int ilk_LA_mode_cal_ena;
> +} cvmx_ilk_LA_mode_t;
> +
> +extern cvmx_ilk_LA_mode_t cvmx_ilk_LA_mode[CVMX_NUM_ILK_INTF];
> +
> +int cvmx_ilk_use_la_mode(int interface, int channel);
> +int cvmx_ilk_start_interface(int interface, unsigned short
> num_lanes);
> +int cvmx_ilk_start_interface_la(int interface, unsigned char
> num_lanes);
> +int cvmx_ilk_set_pipe(int interface, int pipe_base, unsigned int
> pipe_len);
> +int cvmx_ilk_tx_set_channel(int interface, cvmx_ilk_pipe_chan_t
> *pch, unsigned int num_chs);
> +int cvmx_ilk_rx_set_pknd(int interface, cvmx_ilk_chan_pknd_t
> *chpknd, unsigned int num_pknd);
> +int cvmx_ilk_enable(int interface);
> +int cvmx_ilk_disable(int interface);
> +int cvmx_ilk_get_intf_ena(int interface);
> +int cvmx_ilk_get_chan_info(int interface, unsigned char **chans,
> unsigned char *num_chan);
> +cvmx_ilk_la_nsp_compact_hdr_t cvmx_ilk_enable_la_header(int
> ipd_port, int mode);
> +void cvmx_ilk_show_stats(int interface, cvmx_ilk_stats_ctrl_t
> *pstats);
> +int cvmx_ilk_cal_setup_rx(int interface, int cal_depth,
> cvmx_ilk_cal_entry_t *pent, int hi_wm,
> +			  unsigned char cal_ena);
> +int cvmx_ilk_cal_setup_tx(int interface, int cal_depth,
> cvmx_ilk_cal_entry_t *pent,
> +			  unsigned char cal_ena);
> +int cvmx_ilk_lpbk(int interface, cvmx_ilk_lpbk_ena_t enable,
> cvmx_ilk_lpbk_mode_t mode);
> +int cvmx_ilk_la_mode_enable_rx_calendar(int interface);
> +
> +#endif /* __CVMX_ILK_H__ */
> diff --git a/arch/mips/mach-octeon/include/mach/cvmx-ipd.h
> b/arch/mips/mach-octeon/include/mach/cvmx-ipd.h
> new file mode 100644
> index 000000000000..cdff36fffb56
> --- /dev/null
> +++ b/arch/mips/mach-octeon/include/mach/cvmx-ipd.h
> @@ -0,0 +1,233 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +/*
> + * Copyright (C) 2020 Marvell International Ltd.
> + *
> + * Interface to the hardware Input Packet Data unit.
> + */
> +
> +#ifndef __CVMX_IPD_H__
> +#define __CVMX_IPD_H__
> +
> +#include "cvmx-pki.h"
> +
> +/* CSR typedefs have been moved to cvmx-ipd-defs.h */
> +
> +typedef cvmx_ipd_1st_mbuff_skip_t cvmx_ipd_mbuff_not_first_skip_t;
> +typedef cvmx_ipd_1st_next_ptr_back_t
> cvmx_ipd_second_next_ptr_back_t;
> +
> +typedef struct cvmx_ipd_tag_fields {
> +	u64 ipv6_src_ip : 1;
> +	u64 ipv6_dst_ip : 1;
> +	u64 ipv6_src_port : 1;
> +	u64 ipv6_dst_port : 1;
> +	u64 ipv6_next_header : 1;
> +	u64 ipv4_src_ip : 1;
> +	u64 ipv4_dst_ip : 1;
> +	u64 ipv4_src_port : 1;
> +	u64 ipv4_dst_port : 1;
> +	u64 ipv4_protocol : 1;
> +	u64 input_port : 1;
> +} cvmx_ipd_tag_fields_t;
> +
> +typedef struct cvmx_pip_port_config {
> +	u64 parse_mode;
> +	u64 tag_type;
> +	u64 tag_mode;
> +	cvmx_ipd_tag_fields_t tag_fields;
> +} cvmx_pip_port_config_t;
> +
> +typedef struct cvmx_ipd_config_struct {
> +	u64 first_mbuf_skip;
> +	u64 not_first_mbuf_skip;
> +	u64 ipd_enable;
> +	u64 enable_len_M8_fix;
> +	u64 cache_mode;
> +	cvmx_fpa_pool_config_t packet_pool;
> +	cvmx_fpa_pool_config_t wqe_pool;
> +	cvmx_pip_port_config_t port_config;
> +} cvmx_ipd_config_t;
> +
> +extern cvmx_ipd_config_t cvmx_ipd_cfg;
> +
> +/**
> + * Gets the fpa pool number of packet pool
> + */
> +static inline s64 cvmx_fpa_get_packet_pool(void)
> +{
> +	return (cvmx_ipd_cfg.packet_pool.pool_num);
> +}
> +
> +/**
> + * Gets the buffer size of packet pool buffer
> + */
> +static inline u64 cvmx_fpa_get_packet_pool_block_size(void)
> +{
> +	return (cvmx_ipd_cfg.packet_pool.buffer_size);
> +}
> +
> +/**
> + * Gets the buffer count of packet pool
> + */
> +static inline u64 cvmx_fpa_get_packet_pool_buffer_count(void)
> +{
> +	return (cvmx_ipd_cfg.packet_pool.buffer_count);
> +}
> +
> +/**
> + * Gets the fpa pool number of wqe pool
> + */
> +static inline s64 cvmx_fpa_get_wqe_pool(void)
> +{
> +	return (cvmx_ipd_cfg.wqe_pool.pool_num);
> +}
> +
> +/**
> + * Gets the buffer size of wqe pool buffer
> + */
> +static inline u64 cvmx_fpa_get_wqe_pool_block_size(void)
> +{
> +	return (cvmx_ipd_cfg.wqe_pool.buffer_size);
> +}
> +
> +/**
> + * Gets the buffer count of wqe pool
> + */
> +static inline u64 cvmx_fpa_get_wqe_pool_buffer_count(void)
> +{
> +	return (cvmx_ipd_cfg.wqe_pool.buffer_count);
> +}
> +
> +/**
> + * Sets the ipd related configuration in internal structure which is
> then used
> + * for seting IPD hardware block
> + */
> +int cvmx_ipd_set_config(cvmx_ipd_config_t ipd_config);
> +
> +/**
> + * Gets the ipd related configuration from internal structure.
> + */
> +void cvmx_ipd_get_config(cvmx_ipd_config_t *ipd_config);
> +
> +/**
> + * Sets the internal FPA pool data structure for packet buffer pool.
> + * @param pool	fpa pool number yo use
> + * @param buffer_size	buffer size of pool
> + * @param buffer_count	number of buufers to allocate to pool
> + */
> +void cvmx_ipd_set_packet_pool_config(s64 pool, u64 buffer_size, u64
> buffer_count);
> +
> +/**
> + * Sets the internal FPA pool data structure for wqe pool.
> + * @param pool	fpa pool number yo use
> + * @param buffer_size	buffer size of pool
> + * @param buffer_count	number of buufers to allocate to pool
> + */
> +void cvmx_ipd_set_wqe_pool_config(s64 pool, u64 buffer_size, u64
> buffer_count);
> +
> +/**
> + * Gets the FPA packet buffer pool parameters.
> + */
> +static inline void cvmx_fpa_get_packet_pool_config(s64 *pool, u64
> *buffer_size, u64 *buffer_count)
> +{
> +	if (pool)
> +		*pool = cvmx_ipd_cfg.packet_pool.pool_num;
> +	if (buffer_size)
> +		*buffer_size = cvmx_ipd_cfg.packet_pool.buffer_size;
> +	if (buffer_count)
> +		*buffer_count = cvmx_ipd_cfg.packet_pool.buffer_count;
> +}
> +
> +/**
> + * Sets the FPA packet buffer pool parameters.
> + */
> +static inline void cvmx_fpa_set_packet_pool_config(s64 pool, u64
> buffer_size, u64 buffer_count)
> +{
> +	cvmx_ipd_set_packet_pool_config(pool, buffer_size,
> buffer_count);
> +}
> +
> +/**
> + * Gets the FPA WQE pool parameters.
> + */
> +static inline void cvmx_fpa_get_wqe_pool_config(s64 *pool, u64
> *buffer_size, u64 *buffer_count)
> +{
> +	if (pool)
> +		*pool = cvmx_ipd_cfg.wqe_pool.pool_num;
> +	if (buffer_size)
> +		*buffer_size = cvmx_ipd_cfg.wqe_pool.buffer_size;
> +	if (buffer_count)
> +		*buffer_count = cvmx_ipd_cfg.wqe_pool.buffer_count;
> +}
> +
> +/**
> + * Sets the FPA WQE pool parameters.
> + */
> +static inline void cvmx_fpa_set_wqe_pool_config(s64 pool, u64
> buffer_size, u64 buffer_count)
> +{
> +	cvmx_ipd_set_wqe_pool_config(pool, buffer_size, buffer_count);
> +}
> +
> +/**
> + * Configure IPD
> + *
> + * @param mbuff_size Packets buffer size in 8 byte words
> + * @param first_mbuff_skip
> + *                   Number of 8 byte words to skip in the first
> buffer
> + * @param not_first_mbuff_skip
> + *                   Number of 8 byte words to skip in each
> following buffer
> + * @param first_back Must be same as first_mbuff_skip / 128
> + * @param second_back
> + *                   Must be same as not_first_mbuff_skip / 128
> + * @param wqe_fpa_pool
> + *                   FPA pool to get work entries from
> + * @param cache_mode
> + * @param back_pres_enable_flag
> + *                   Enable or disable port back pressure at a
> global level.
> + *                   This should always be 1 as more accurate
> control can be
> + *                   found in IPD_PORTX_BP_PAGE_CNT[BP_ENB].
> + */
> +void cvmx_ipd_config(u64 mbuff_size, u64 first_mbuff_skip, u64
> not_first_mbuff_skip, u64 first_back,
> +		     u64 second_back, u64 wqe_fpa_pool, cvmx_ipd_mode_t
> cache_mode,
> +		     u64 back_pres_enable_flag);
> +/**
> + * Enable IPD
> + */
> +void cvmx_ipd_enable(void);
> +
> +/**
> + * Disable IPD
> + */
> +void cvmx_ipd_disable(void);
> +
> +void __cvmx_ipd_free_ptr(void);
> +
> +void cvmx_ipd_set_packet_pool_buffer_count(u64 buffer_count);
> +void cvmx_ipd_set_wqe_pool_buffer_count(u64 buffer_count);
> +
> +/**
> + * Setup Random Early Drop on a specific input queue
> + *
> + * @param queue  Input queue to setup RED on (0-7)
> + * @param pass_thresh
> + *               Packets will begin slowly dropping when there are
> less than
> + *               this many packet buffers free in FPA 0.
> + * @param drop_thresh
> + *               All incoming packets will be dropped when there are
> less
> + *               than this many free packet buffers in FPA 0.
> + * @return Zero on success. Negative on failure
> + */
> +int cvmx_ipd_setup_red_queue(int queue, int pass_thresh, int
> drop_thresh);
> +
> +/**
> + * Setup Random Early Drop to automatically begin dropping packets.
> + *
> + * @param pass_thresh
> + *               Packets will begin slowly dropping when there are
> less than
> + *               this many packet buffers free in FPA 0.
> + * @param drop_thresh
> + *               All incoming packets will be dropped when there are
> less
> + *               than this many free packet buffers in FPA 0.
> + * @return Zero on success. Negative on failure
> + */
> +int cvmx_ipd_setup_red(int pass_thresh, int drop_thresh);
> +
> +#endif /*  __CVMX_IPD_H__ */
> diff --git a/arch/mips/mach-octeon/include/mach/cvmx-packet.h
> b/arch/mips/mach-octeon/include/mach/cvmx-packet.h
> new file mode 100644
> index 000000000000..f3cfe9c64f43
> --- /dev/null
> +++ b/arch/mips/mach-octeon/include/mach/cvmx-packet.h
> @@ -0,0 +1,40 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +/*
> + * Copyright (C) 2020 Marvell International Ltd.
> + *
> + * Packet buffer defines.
> + */
> +
> +#ifndef __CVMX_PACKET_H__
> +#define __CVMX_PACKET_H__
> +
> +union cvmx_buf_ptr_pki {
> +	u64 u64;
> +	struct {
> +		u64 size : 16;
> +		u64 packet_outside_wqe : 1;
> +		u64 rsvd0 : 5;
> +		u64 addr : 42;
> +	};
> +};
> +
> +typedef union cvmx_buf_ptr_pki cvmx_buf_ptr_pki_t;
> +
> +/**
> + * This structure defines a buffer pointer on Octeon
> + */
> +union cvmx_buf_ptr {
> +	void *ptr;
> +	u64 u64;
> +	struct {
> +		u64 i : 1;
> +		u64 back : 4;
> +		u64 pool : 3;
> +		u64 size : 16;
> +		u64 addr : 40;
> +	} s;
> +};
> +
> +typedef union cvmx_buf_ptr cvmx_buf_ptr_t;
> +
> +#endif /*  __CVMX_PACKET_H__ */
> diff --git a/arch/mips/mach-octeon/include/mach/cvmx-pcie.h
> b/arch/mips/mach-octeon/include/mach/cvmx-pcie.h
> new file mode 100644
> index 000000000000..a819196c021c
> --- /dev/null
> +++ b/arch/mips/mach-octeon/include/mach/cvmx-pcie.h
> @@ -0,0 +1,279 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +/*
> + * Copyright (C) 2020 Marvell International Ltd.
> + */
> +
> +#ifndef __CVMX_PCIE_H__
> +#define __CVMX_PCIE_H__
> +
> +#define CVMX_PCIE_MAX_PORTS 4
> +#define
> CVMX_PCIE_PORTS                                                      
>                       \
> +	((OCTEON_IS_MODEL(OCTEON_CN78XX) ||
> OCTEON_IS_MODEL(OCTEON_CN73XX)) ?                      \
> +		       CVMX_PCIE_MAX_PORTS
> :                                                             \
> +		       (OCTEON_IS_MODEL(OCTEON_CN70XX) ? 3 : 2))
> +
> +/*
> + * The physical memory base mapped by BAR1.  256MB at the end of the
> + * first 4GB.
> + */
> +#define CVMX_PCIE_BAR1_PHYS_BASE ((1ull << 32) - (1ull << 28))
> +#define CVMX_PCIE_BAR1_PHYS_SIZE BIT_ULL(28)
> +
> +/*
> + * The RC base of BAR1.  gen1 has a 39-bit BAR2, gen2 has 41-bit
> BAR2,
> + * place BAR1 so it is the same for both.
> + */
> +#define CVMX_PCIE_BAR1_RC_BASE BIT_ULL(41)
> +
> +typedef union {
> +	u64 u64;
> +	struct {
> +		u64 upper : 2;		 /* Normally 2 for XKPHYS */
> +		u64 reserved_49_61 : 13; /* Must be zero */
> +		u64 io : 1;		 /* 1 for IO space access */
> +		u64 did : 5;		 /* PCIe DID = 3 */
> +		u64 subdid : 3;		 /* PCIe SubDID = 1 */
> +		u64 reserved_38_39 : 2;	 /* Must be zero */
> +		u64 node : 2;		 /* Numa node number */
> +		u64 es : 2;		 /* Endian swap = 1 */
> +		u64 port : 2;		 /* PCIe port 0,1 */
> +		u64 reserved_29_31 : 3;	 /* Must be zero */
> +		u64 ty : 1;
> +		u64 bus : 8;
> +		u64 dev : 5;
> +		u64 func : 3;
> +		u64 reg : 12;
> +	} config;
> +	struct {
> +		u64 upper : 2;		 /* Normally 2 for XKPHYS */
> +		u64 reserved_49_61 : 13; /* Must be zero */
> +		u64 io : 1;		 /* 1 for IO space access */
> +		u64 did : 5;		 /* PCIe DID = 3 */
> +		u64 subdid : 3;		 /* PCIe SubDID = 2 */
> +		u64 reserved_38_39 : 2;	 /* Must be zero */
> +		u64 node : 2;		 /* Numa node number */
> +		u64 es : 2;		 /* Endian swap = 1 */
> +		u64 port : 2;		 /* PCIe port 0,1 */
> +		u64 address : 32;	 /* PCIe IO address */
> +	} io;
> +	struct {
> +		u64 upper : 2;		 /* Normally 2 for XKPHYS */
> +		u64 reserved_49_61 : 13; /* Must be zero */
> +		u64 io : 1;		 /* 1 for IO space access */
> +		u64 did : 5;		 /* PCIe DID = 3 */
> +		u64 subdid : 3;		 /* PCIe SubDID = 3-6 */
> +		u64 reserved_38_39 : 2;	 /* Must be zero */
> +		u64 node : 2;		 /* Numa node number */
> +		u64 address : 36;	 /* PCIe Mem address */
> +	} mem;
> +} cvmx_pcie_address_t;
> +
> +/**
> + * Return the Core virtual base address for PCIe IO access. IOs are
> + * read/written as an offset from this address.
> + *
> + * @param pcie_port PCIe port the IO is for
> + *
> + * @return 64bit Octeon IO base address for read/write
> + */
> +u64 cvmx_pcie_get_io_base_address(int pcie_port);
> +
> +/**
> + * Size of the IO address region returned at address
> + * cvmx_pcie_get_io_base_address()
> + *
> + * @param pcie_port PCIe port the IO is for
> + *
> + * @return Size of the IO window
> + */
> +u64 cvmx_pcie_get_io_size(int pcie_port);
> +
> +/**
> + * Return the Core virtual base address for PCIe MEM access. Memory
> is
> + * read/written as an offset from this address.
> + *
> + * @param pcie_port PCIe port the IO is for
> + *
> + * @return 64bit Octeon IO base address for read/write
> + */
> +u64 cvmx_pcie_get_mem_base_address(int pcie_port);
> +
> +/**
> + * Size of the Mem address region returned at address
> + * cvmx_pcie_get_mem_base_address()
> + *
> + * @param pcie_port PCIe port the IO is for
> + *
> + * @return Size of the Mem window
> + */
> +u64 cvmx_pcie_get_mem_size(int pcie_port);
> +
> +/**
> + * Initialize a PCIe port for use in host(RC) mode. It doesn't
> enumerate the bus.
> + *
> + * @param pcie_port PCIe port to initialize
> + *
> + * @return Zero on success
> + */
> +int cvmx_pcie_rc_initialize(int pcie_port);
> +
> +/**
> + * Shutdown a PCIe port and put it in reset
> + *
> + * @param pcie_port PCIe port to shutdown
> + *
> + * @return Zero on success
> + */
> +int cvmx_pcie_rc_shutdown(int pcie_port);
> +
> +/**
> + * Read 8bits from a Device's config space
> + *
> + * @param pcie_port PCIe port the device is on
> + * @param bus       Sub bus
> + * @param dev       Device ID
> + * @param fn        Device sub function
> + * @param reg       Register to access
> + *
> + * @return Result of the read
> + */
> +u8 cvmx_pcie_config_read8(int pcie_port, int bus, int dev, int fn,
> int reg);
> +
> +/**
> + * Read 16bits from a Device's config space
> + *
> + * @param pcie_port PCIe port the device is on
> + * @param bus       Sub bus
> + * @param dev       Device ID
> + * @param fn        Device sub function
> + * @param reg       Register to access
> + *
> + * @return Result of the read
> + */
> +u16 cvmx_pcie_config_read16(int pcie_port, int bus, int dev, int fn,
> int reg);
> +
> +/**
> + * Read 32bits from a Device's config space
> + *
> + * @param pcie_port PCIe port the device is on
> + * @param bus       Sub bus
> + * @param dev       Device ID
> + * @param fn        Device sub function
> + * @param reg       Register to access
> + *
> + * @return Result of the read
> + */
> +u32 cvmx_pcie_config_read32(int pcie_port, int bus, int dev, int fn,
> int reg);
> +
> +/**
> + * Write 8bits to a Device's config space
> + *
> + * @param pcie_port PCIe port the device is on
> + * @param bus       Sub bus
> + * @param dev       Device ID
> + * @param fn        Device sub function
> + * @param reg       Register to access
> + * @param val       Value to write
> + */
> +void cvmx_pcie_config_write8(int pcie_port, int bus, int dev, int
> fn, int reg, u8 val);
> +
> +/**
> + * Write 16bits to a Device's config space
> + *
> + * @param pcie_port PCIe port the device is on
> + * @param bus       Sub bus
> + * @param dev       Device ID
> + * @param fn        Device sub function
> + * @param reg       Register to access
> + * @param val       Value to write
> + */
> +void cvmx_pcie_config_write16(int pcie_port, int bus, int dev, int
> fn, int reg, u16 val);
> +
> +/**
> + * Write 32bits to a Device's config space
> + *
> + * @param pcie_port PCIe port the device is on
> + * @param bus       Sub bus
> + * @param dev       Device ID
> + * @param fn        Device sub function
> + * @param reg       Register to access
> + * @param val       Value to write
> + */
> +void cvmx_pcie_config_write32(int pcie_port, int bus, int dev, int
> fn, int reg, u32 val);
> +
> +/**
> + * Read a PCIe config space register indirectly. This is used for
> + * registers of the form PCIEEP_CFG??? and PCIERC?_CFG???.
> + *
> + * @param pcie_port  PCIe port to read from
> + * @param cfg_offset Address to read
> + *
> + * @return Value read
> + */
> +u32 cvmx_pcie_cfgx_read(int pcie_port, u32 cfg_offset);
> +u32 cvmx_pcie_cfgx_read_node(int node, int pcie_port, u32
> cfg_offset);
> +
> +/**
> + * Write a PCIe config space register indirectly. This is used for
> + * registers of the form PCIEEP_CFG??? and PCIERC?_CFG???.
> + *
> + * @param pcie_port  PCIe port to write to
> + * @param cfg_offset Address to write
> + * @param val        Value to write
> + */
> +void cvmx_pcie_cfgx_write(int pcie_port, u32 cfg_offset, u32 val);
> +void cvmx_pcie_cfgx_write_node(int node, int pcie_port, u32
> cfg_offset, u32 val);
> +
> +/**
> + * Write a 32bit value to the Octeon NPEI register space
> + *
> + * @param address Address to write to
> + * @param val     Value to write
> + */
> +static inline void cvmx_pcie_npei_write32(u64 address, u32 val)
> +{
> +	cvmx_write64_uint32(address ^ 4, val);
> +	cvmx_read64_uint32(address ^ 4);
> +}
> +
> +/**
> + * Read a 32bit value from the Octeon NPEI register space
> + *
> + * @param address Address to read
> + * @return The result
> + */
> +static inline u32 cvmx_pcie_npei_read32(u64 address)
> +{
> +	return cvmx_read64_uint32(address ^ 4);
> +}
> +
> +/**
> + * Initialize a PCIe port for use in target(EP) mode.
> + *
> + * @param pcie_port PCIe port to initialize
> + *
> + * @return Zero on success
> + */
> +int cvmx_pcie_ep_initialize(int pcie_port);
> +
> +/**
> + * Wait for posted PCIe read/writes to reach the other side of
> + * the internal PCIe switch. This will insure that core
> + * read/writes are posted before anything after this function
> + * is called. This may be necessary when writing to memory that
> + * will later be read using the DMA/PKT engines.
> + *
> + * @param pcie_port PCIe port to wait for
> + */
> +void cvmx_pcie_wait_for_pending(int pcie_port);
> +
> +/**
> + * Returns if a PCIe port is in host or target mode.
> + *
> + * @param pcie_port PCIe port number (PEM number)
> + *
> + * @return 0 if PCIe port is in target mode, !0 if in host mode.
> + */
> +int cvmx_pcie_is_host_mode(int pcie_port);
> +
> +#endif
> diff --git a/arch/mips/mach-octeon/include/mach/cvmx-pip.h
> b/arch/mips/mach-octeon/include/mach/cvmx-pip.h
> new file mode 100644
> index 000000000000..013f533fb7bb
> --- /dev/null
> +++ b/arch/mips/mach-octeon/include/mach/cvmx-pip.h
> @@ -0,0 +1,1080 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +/*
> + * Copyright (C) 2020 Marvell International Ltd.
> + *
> + * Interface to the hardware Packet Input Processing unit.
> + */
> +
> +#ifndef __CVMX_PIP_H__
> +#define __CVMX_PIP_H__
> +
> +#include "cvmx-wqe.h"
> +#include "cvmx-pki.h"
> +#include "cvmx-helper-pki.h"
> +
> +#include "cvmx-helper.h"
> +#include "cvmx-helper-util.h"
> +#include "cvmx-pki-resources.h"
> +
> +#define CVMX_PIP_NUM_INPUT_PORTS 46
> +#define CVMX_PIP_NUM_WATCHERS	 8
> +
> +/*
> + * Encodes the different error and exception codes
> + */
> +typedef enum {
> +	CVMX_PIP_L4_NO_ERR = 0ull,
> +	/*        1  = TCP (UDP) packet not long enough to cover TCP
> (UDP) header */
> +	CVMX_PIP_L4_MAL_ERR = 1ull,
> +	/*        2  = TCP/UDP checksum failure */
> +	CVMX_PIP_CHK_ERR = 2ull,
> +	/*        3  = TCP/UDP length check (TCP/UDP length does not
> match IP length) */
> +	CVMX_PIP_L4_LENGTH_ERR = 3ull,
> +	/*        4  = illegal TCP/UDP port (either source or dest port
> is zero) */
> +	CVMX_PIP_BAD_PRT_ERR = 4ull,
> +	/*        8  = TCP flags = FIN only */
> +	CVMX_PIP_TCP_FLG8_ERR = 8ull,
> +	/*        9  = TCP flags = 0 */
> +	CVMX_PIP_TCP_FLG9_ERR = 9ull,
> +	/*        10 = TCP flags = FIN+RST+* */
> +	CVMX_PIP_TCP_FLG10_ERR = 10ull,
> +	/*        11 = TCP flags = SYN+URG+* */
> +	CVMX_PIP_TCP_FLG11_ERR = 11ull,
> +	/*        12 = TCP flags = SYN+RST+* */
> +	CVMX_PIP_TCP_FLG12_ERR = 12ull,
> +	/*        13 = TCP flags = SYN+FIN+* */
> +	CVMX_PIP_TCP_FLG13_ERR = 13ull
> +} cvmx_pip_l4_err_t;
> +
> +typedef enum {
> +	CVMX_PIP_IP_NO_ERR = 0ull,
> +	/*        1 = not IPv4 or IPv6 */
> +	CVMX_PIP_NOT_IP = 1ull,
> +	/*        2 = IPv4 header checksum violation */
> +	CVMX_PIP_IPV4_HDR_CHK = 2ull,
> +	/*        3 = malformed (packet not long enough to cover IP
> hdr) */
> +	CVMX_PIP_IP_MAL_HDR = 3ull,
> +	/*        4 = malformed (packet not long enough to cover len in
> IP hdr) */
> +	CVMX_PIP_IP_MAL_PKT = 4ull,
> +	/*        5 = TTL / hop count equal zero */
> +	CVMX_PIP_TTL_HOP = 5ull,
> +	/*        6 = IPv4 options / IPv6 early extension headers */
> +	CVMX_PIP_OPTS = 6ull
> +} cvmx_pip_ip_exc_t;
> +
> +/**
> + * NOTES
> + *       late collision (data received before collision)
> + *            late collisions cannot be detected by the receiver
> + *            they would appear as JAM bits which would appear as
> bad FCS
> + *            or carrier extend error which is CVMX_PIP_EXTEND_ERR
> + */
> +typedef enum {
> +	/**
> +	 * No error
> +	 */
> +	CVMX_PIP_RX_NO_ERR = 0ull,
> +
> +	CVMX_PIP_PARTIAL_ERR =
> +		1ull, /* RGM+SPI            1 = partially received
> packet (buffering/bandwidth not adequate) */
> +	CVMX_PIP_JABBER_ERR =
> +		2ull, /* RGM+SPI            2 = receive packet too
> large and truncated */
> +	CVMX_PIP_OVER_FCS_ERR =
> +		3ull, /* RGM                3 = max frame error (pkt
> len > max frame len) (with FCS error) */
> +	CVMX_PIP_OVER_ERR =
> +		4ull, /* RGM+SPI            4 = max frame error (pkt
> len > max frame len) */
> +	CVMX_PIP_ALIGN_ERR =
> +		5ull, /* RGM                5 = nibble error (data not
> byte multiple - 100M and 10M only) */
> +	CVMX_PIP_UNDER_FCS_ERR =
> +		6ull, /* RGM                6 = min frame error (pkt
> len < min frame len) (with FCS error) */
> +	CVMX_PIP_GMX_FCS_ERR = 7ull, /* RGM                7 = FCS
> error */
> +	CVMX_PIP_UNDER_ERR =
> +		8ull, /* RGM+SPI            8 = min frame error (pkt
> len < min frame len) */
> +	CVMX_PIP_EXTEND_ERR = 9ull, /* RGM                9 = Frame
> carrier extend error */
> +	CVMX_PIP_TERMINATE_ERR =
> +		9ull, /* XAUI               9 = Packet was terminated
> with an idle cycle */
> +	CVMX_PIP_LENGTH_ERR =
> +		10ull, /* RGM               10 = length mismatch (len
> did not match len in L2 length/type) */
> +	CVMX_PIP_DAT_ERR =
> +		11ull, /* RGM               11 = Frame error (some or
> all data bits marked err) */
> +	CVMX_PIP_DIP_ERR = 11ull, /*     SPI           11 = DIP4 error
> */
> +	CVMX_PIP_SKIP_ERR =
> +		12ull, /* RGM               12 = packet was not large
> enough to pass the skipper - no inspection could occur */
> +	CVMX_PIP_NIBBLE_ERR =
> +		13ull, /* RGM               13 = studder error (data
> not repeated - 100M and 10M only) */
> +	CVMX_PIP_PIP_FCS = 16L, /* RGM+SPI           16 = FCS error */
> +	CVMX_PIP_PIP_SKIP_ERR =
> +		17L, /* RGM+SPI+PCI       17 = packet was not large
> enough to pass the skipper - no inspection could occur */
> +	CVMX_PIP_PIP_L2_MAL_HDR =
> +		18L, /* RGM+SPI+PCI       18 = malformed l2 (packet not
> long enough to cover L2 hdr) */
> +	CVMX_PIP_PUNY_ERR =
> +		47L /* SGMII             47 = PUNY error (packet was 4B
> or less when FCS stripping is enabled) */
> +	/* NOTES
> +	 *       xx = late collision (data received before collision)
> +	 *            late collisions cannot be detected by the
> receiver
> +	 *            they would appear as JAM bits which would appear
> as bad FCS
> +	 *            or carrier extend error which is
> CVMX_PIP_EXTEND_ERR
> +	 */
> +} cvmx_pip_rcv_err_t;
> +
> +/**
> + * This defines the err_code field errors in the work Q entry
> + */
> +typedef union {
> +	cvmx_pip_l4_err_t l4_err;
> +	cvmx_pip_ip_exc_t ip_exc;
> +	cvmx_pip_rcv_err_t rcv_err;
> +} cvmx_pip_err_t;
> +
> +/**
> + * Status statistics for a port
> + */
> +typedef struct {
> +	u64 dropped_octets;
> +	u64 dropped_packets;
> +	u64 pci_raw_packets;
> +	u64 octets;
> +	u64 packets;
> +	u64 multicast_packets;
> +	u64 broadcast_packets;
> +	u64 len_64_packets;
> +	u64 len_65_127_packets;
> +	u64 len_128_255_packets;
> +	u64 len_256_511_packets;
> +	u64 len_512_1023_packets;
> +	u64 len_1024_1518_packets;
> +	u64 len_1519_max_packets;
> +	u64 fcs_align_err_packets;
> +	u64 runt_packets;
> +	u64 runt_crc_packets;
> +	u64 oversize_packets;
> +	u64 oversize_crc_packets;
> +	u64 inb_packets;
> +	u64 inb_octets;
> +	u64 inb_errors;
> +	u64 mcast_l2_red_packets;
> +	u64 bcast_l2_red_packets;
> +	u64 mcast_l3_red_packets;
> +	u64 bcast_l3_red_packets;
> +} cvmx_pip_port_status_t;
> +
> +/**
> + * Definition of the PIP custom header that can be prepended
> + * to a packet by external hardware.
> + */
> +typedef union {
> +	u64 u64;
> +	struct {
> +		u64 rawfull : 1;
> +		u64 reserved0 : 5;
> +		cvmx_pip_port_parse_mode_t parse_mode : 2;
> +		u64 reserved1 : 1;
> +		u64 skip_len : 7;
> +		u64 grpext : 2;
> +		u64 nqos : 1;
> +		u64 ngrp : 1;
> +		u64 ntt : 1;
> +		u64 ntag : 1;
> +		u64 qos : 3;
> +		u64 grp : 4;
> +		u64 rs : 1;
> +		cvmx_pow_tag_type_t tag_type : 2;
> +		u64 tag : 32;
> +	} s;
> +} cvmx_pip_pkt_inst_hdr_t;
> +
> +enum cvmx_pki_pcam_match {
> +	CVMX_PKI_PCAM_MATCH_IP,
> +	CVMX_PKI_PCAM_MATCH_IPV4,
> +	CVMX_PKI_PCAM_MATCH_IPV6,
> +	CVMX_PKI_PCAM_MATCH_TCP
> +};
> +
> +/* CSR typedefs have been moved to cvmx-pip-defs.h */
> +static inline int cvmx_pip_config_watcher(int index, int type, u16
> match, u16 mask, int grp,
> +					  int qos)
> +{
> +	if (index >= CVMX_PIP_NUM_WATCHERS) {
> +		debug("ERROR: pip watcher %d is > than supported\n",
> index);
> +		return -1;
> +	}
> +	if (octeon_has_feature(OCTEON_FEATURE_PKI)) {
> +		/* store in software for now, only when the watcher is
> enabled program the entry*/
> +		if (type == CVMX_PIP_QOS_WATCH_PROTNH) {
> +			qos_watcher[index].field =
> CVMX_PKI_PCAM_TERM_L3_FLAGS;
> +			qos_watcher[index].data = (u32)(match << 16);
> +			qos_watcher[index].data_mask = (u32)(mask <<
> 16);
> +			qos_watcher[index].advance = 0;
> +		} else if (type == CVMX_PIP_QOS_WATCH_TCP) {
> +			qos_watcher[index].field =
> CVMX_PKI_PCAM_TERM_L4_PORT;
> +			qos_watcher[index].data = 0x060000;
> +			qos_watcher[index].data |= (u32)match;
> +			qos_watcher[index].data_mask = (u32)(mask);
> +			qos_watcher[index].advance = 0;
> +		} else if (type == CVMX_PIP_QOS_WATCH_UDP) {
> +			qos_watcher[index].field =
> CVMX_PKI_PCAM_TERM_L4_PORT;
> +			qos_watcher[index].data = 0x110000;
> +			qos_watcher[index].data |= (u32)match;
> +			qos_watcher[index].data_mask = (u32)(mask);
> +			qos_watcher[index].advance = 0;
> +		} else if (type == 0x4
> /*CVMX_PIP_QOS_WATCH_ETHERTYPE*/) {
> +			qos_watcher[index].field =
> CVMX_PKI_PCAM_TERM_ETHTYPE0;
> +			if (match == 0x8100) {
> +				debug("ERROR: default vlan entry
> already exist, cant set watcher\n");
> +				return -1;
> +			}
> +			qos_watcher[index].data = (u32)(match << 16);
> +			qos_watcher[index].data_mask = (u32)(mask <<
> 16);
> +			qos_watcher[index].advance = 4;
> +		} else {
> +			debug("ERROR: Unsupported watcher type %d\n",
> type);
> +			return -1;
> +		}
> +		if (grp >= 32) {
> +			debug("ERROR: grp %d out of range for backward
> compat 78xx\n", grp);
> +			return -1;
> +		}
> +		qos_watcher[index].sso_grp = (u8)(grp << 3 | qos);
> +		qos_watcher[index].configured = 1;
> +	} else {
> +		/* Implement it later */
> +	}
> +	return 0;
> +}
> +
> +static inline int __cvmx_pip_set_tag_type(int node, int style, int
> tag_type, int field)
> +{
> +	struct cvmx_pki_style_config style_cfg;
> +	int style_num;
> +	int pcam_offset;
> +	int bank;
> +	struct cvmx_pki_pcam_input pcam_input;
> +	struct cvmx_pki_pcam_action pcam_action;
> +
> +	/* All other style parameters remain same except tag type */
> +	cvmx_pki_read_style_config(node, style, CVMX_PKI_CLUSTER_ALL,
> &style_cfg);
> +	style_cfg.parm_cfg.tag_type = (enum cvmx_sso_tag_type)tag_type;
> +	style_num = cvmx_pki_style_alloc(node, -1);
> +	if (style_num < 0) {
> +		debug("ERROR: style not available to set tag type\n");
> +		return -1;
> +	}
> +	cvmx_pki_write_style_config(node, style_num,
> CVMX_PKI_CLUSTER_ALL, &style_cfg);
> +	memset(&pcam_input, 0, sizeof(pcam_input));
> +	memset(&pcam_action, 0, sizeof(pcam_action));
> +	pcam_input.style = style;
> +	pcam_input.style_mask = 0xff;
> +	if (field == CVMX_PKI_PCAM_MATCH_IP) {
> +		pcam_input.field = CVMX_PKI_PCAM_TERM_ETHTYPE0;
> +		pcam_input.field_mask = 0xff;
> +		pcam_input.data = 0x08000000;
> +		pcam_input.data_mask = 0xffff0000;
> +		pcam_action.pointer_advance = 4;
> +		/* legacy will write to all clusters*/
> +		bank = 0;
> +		pcam_offset = cvmx_pki_pcam_entry_alloc(node,
> CVMX_PKI_FIND_AVAL_ENTRY, bank,
> +							CVMX_PKI_CLUSTE
> R_ALL);
> +		if (pcam_offset < 0) {
> +			debug("ERROR: pcam entry not available to
> enable qos watcher\n");
> +			cvmx_pki_style_free(node, style_num);
> +			return -1;
> +		}
> +		pcam_action.parse_mode_chg = CVMX_PKI_PARSE_NO_CHG;
> +		pcam_action.layer_type_set = CVMX_PKI_LTYPE_E_NONE;
> +		pcam_action.style_add = (u8)(style_num - style);
> +		cvmx_pki_pcam_write_entry(node, pcam_offset,
> CVMX_PKI_CLUSTER_ALL, pcam_input,
> +					  pcam_action);
> +		field = CVMX_PKI_PCAM_MATCH_IPV6;
> +	}
> +	if (field == CVMX_PKI_PCAM_MATCH_IPV4) {
> +		pcam_input.field = CVMX_PKI_PCAM_TERM_ETHTYPE0;
> +		pcam_input.field_mask = 0xff;
> +		pcam_input.data = 0x08000000;
> +		pcam_input.data_mask = 0xffff0000;
> +		pcam_action.pointer_advance = 4;
> +	} else if (field == CVMX_PKI_PCAM_MATCH_IPV6) {
> +		pcam_input.field = CVMX_PKI_PCAM_TERM_ETHTYPE0;
> +		pcam_input.field_mask = 0xff;
> +		pcam_input.data = 0x86dd00000;
> +		pcam_input.data_mask = 0xffff0000;
> +		pcam_action.pointer_advance = 4;
> +	} else if (field == CVMX_PKI_PCAM_MATCH_TCP) {
> +		pcam_input.field = CVMX_PKI_PCAM_TERM_L4_PORT;
> +		pcam_input.field_mask = 0xff;
> +		pcam_input.data = 0x60000;
> +		pcam_input.data_mask = 0xff0000;
> +		pcam_action.pointer_advance = 0;
> +	}
> +	pcam_action.parse_mode_chg = CVMX_PKI_PARSE_NO_CHG;
> +	pcam_action.layer_type_set = CVMX_PKI_LTYPE_E_NONE;
> +	pcam_action.style_add = (u8)(style_num - style);
> +	bank = pcam_input.field & 0x01;
> +	pcam_offset = cvmx_pki_pcam_entry_alloc(node,
> CVMX_PKI_FIND_AVAL_ENTRY, bank,
> +						CVMX_PKI_CLUSTER_ALL);
> +	if (pcam_offset < 0) {
> +		debug("ERROR: pcam entry not available to enable qos
> watcher\n");
> +		cvmx_pki_style_free(node, style_num);
> +		return -1;
> +	}
> +	cvmx_pki_pcam_write_entry(node, pcam_offset,
> CVMX_PKI_CLUSTER_ALL, pcam_input, pcam_action);
> +	return style_num;
> +}
> +
> +/* Only for legacy internal use */
> +static inline int __cvmx_pip_enable_watcher_78xx(int node, int
> index, int style)
> +{
> +	struct cvmx_pki_style_config style_cfg;
> +	struct cvmx_pki_qpg_config qpg_cfg;
> +	struct cvmx_pki_pcam_input pcam_input;
> +	struct cvmx_pki_pcam_action pcam_action;
> +	int style_num;
> +	int qpg_offset;
> +	int pcam_offset;
> +	int bank;
> +
> +	if (!qos_watcher[index].configured) {
> +		debug("ERROR: qos watcher %d should be configured
> before enable\n", index);
> +		return -1;
> +	}
> +	/* All other style parameters remain same except grp and qos
> and qps base */
> +	cvmx_pki_read_style_config(node, style, CVMX_PKI_CLUSTER_ALL,
> &style_cfg);
> +	cvmx_pki_read_qpg_entry(node, style_cfg.parm_cfg.qpg_base,
> &qpg_cfg);
> +	qpg_cfg.qpg_base = CVMX_PKI_FIND_AVAL_ENTRY;
> +	qpg_cfg.grp_ok = qos_watcher[index].sso_grp;
> +	qpg_cfg.grp_bad = qos_watcher[index].sso_grp;
> +	qpg_offset = cvmx_helper_pki_set_qpg_entry(node, &qpg_cfg);
> +	if (qpg_offset == -1) {
> +		debug("Warning: no new qpg entry available to enable
> watcher\n");
> +		return -1;
> +	}
> +	/* try to reserve the style, if it is not configured already,
> reserve
> +	   and configure it */
> +	style_cfg.parm_cfg.qpg_base = qpg_offset;
> +	style_num = cvmx_pki_style_alloc(node, -1);
> +	if (style_num < 0) {
> +		debug("ERROR: style not available to enable qos
> watcher\n");
> +		cvmx_pki_qpg_entry_free(node, qpg_offset, 1);
> +		return -1;
> +	}
> +	cvmx_pki_write_style_config(node, style_num,
> CVMX_PKI_CLUSTER_ALL, &style_cfg);
> +	/* legacy will write to all clusters*/
> +	bank = qos_watcher[index].field & 0x01;
> +	pcam_offset = cvmx_pki_pcam_entry_alloc(node,
> CVMX_PKI_FIND_AVAL_ENTRY, bank,
> +						CVMX_PKI_CLUSTER_ALL);
> +	if (pcam_offset < 0) {
> +		debug("ERROR: pcam entry not available to enable qos
> watcher\n");
> +		cvmx_pki_style_free(node, style_num);
> +		cvmx_pki_qpg_entry_free(node, qpg_offset, 1);
> +		return -1;
> +	}
> +	memset(&pcam_input, 0, sizeof(pcam_input));
> +	memset(&pcam_action, 0, sizeof(pcam_action));
> +	pcam_input.style = style;
> +	pcam_input.style_mask = 0xff;
> +	pcam_input.field = qos_watcher[index].field;
> +	pcam_input.field_mask = 0xff;
> +	pcam_input.data = qos_watcher[index].data;
> +	pcam_input.data_mask = qos_watcher[index].data_mask;
> +	pcam_action.parse_mode_chg = CVMX_PKI_PARSE_NO_CHG;
> +	pcam_action.layer_type_set = CVMX_PKI_LTYPE_E_NONE;
> +	pcam_action.style_add = (u8)(style_num - style);
> +	pcam_action.pointer_advance = qos_watcher[index].advance;
> +	cvmx_pki_pcam_write_entry(node, pcam_offset,
> CVMX_PKI_CLUSTER_ALL, pcam_input, pcam_action);
> +	return 0;
> +}
> +
> +/**
> + * Configure an ethernet input port
> + *
> + * @param ipd_port Port number to configure
> + * @param port_cfg Port hardware configuration
> + * @param port_tag_cfg Port POW tagging configuration
> + */
> +static inline void cvmx_pip_config_port(u64 ipd_port,
> cvmx_pip_prt_cfgx_t port_cfg,
> +					cvmx_pip_prt_tagx_t
> port_tag_cfg)
> +{
> +	struct cvmx_pki_qpg_config qpg_cfg;
> +	int qpg_offset;
> +	u8 tcp_tag = 0xff;
> +	u8 ip_tag = 0xaa;
> +	int style, nstyle, n4style, n6style;
> +
> +	if (octeon_has_feature(OCTEON_FEATURE_PKI)) {
> +		struct cvmx_pki_port_config pki_prt_cfg;
> +		struct cvmx_xport xp =
> cvmx_helper_ipd_port_to_xport(ipd_port);
> +
> +		cvmx_pki_get_port_config(ipd_port, &pki_prt_cfg);
> +		style = pki_prt_cfg.pkind_cfg.initial_style;
> +		if (port_cfg.s.ih_pri || port_cfg.s.vlan_len ||
> port_cfg.s.pad_len)
> +			debug("Warning: 78xx: use different config for
> this option\n");
> +		pki_prt_cfg.style_cfg.parm_cfg.minmax_sel =
> port_cfg.s.len_chk_sel;
> +		pki_prt_cfg.style_cfg.parm_cfg.lenerr_en =
> port_cfg.s.lenerr_en;
> +		pki_prt_cfg.style_cfg.parm_cfg.maxerr_en =
> port_cfg.s.maxerr_en;
> +		pki_prt_cfg.style_cfg.parm_cfg.minerr_en =
> port_cfg.s.minerr_en;
> +		pki_prt_cfg.style_cfg.parm_cfg.fcs_chk =
> port_cfg.s.crc_en;
> +		if (port_cfg.s.grp_wat || port_cfg.s.qos_wat ||
> port_cfg.s.grp_wat_47 ||
> +		    port_cfg.s.qos_wat_47) {
> +			u8 group_mask = (u8)(port_cfg.s.grp_wat |
> (u8)(port_cfg.s.grp_wat_47 << 4));
> +			u8 qos_mask = (u8)(port_cfg.s.qos_wat |
> (u8)(port_cfg.s.qos_wat_47 << 4));
> +			int i;
> +
> +			for (i = 0; i < CVMX_PIP_NUM_WATCHERS; i++) {
> +				if ((group_mask & (1 << i)) ||
> (qos_mask & (1 << i)))
> +					__cvmx_pip_enable_watcher_78xx(
> xp.node, i, style);
> +			}
> +		}
> +		if (port_tag_cfg.s.tag_mode) {
> +			if (OCTEON_IS_MODEL(OCTEON_CN78XX_PASS1_X))
> +				cvmx_printf("Warning: mask tag is not
> supported in 78xx pass1\n");
> +			else {
> +			}
> +			/* need to implement for 78xx*/
> +		}
> +		if (port_cfg.s.tag_inc)
> +			debug("Warning: 78xx uses differnet method for
> tag generation\n");
> +		pki_prt_cfg.style_cfg.parm_cfg.rawdrp =
> port_cfg.s.rawdrp;
> +		pki_prt_cfg.pkind_cfg.parse_en.inst_hdr =
> port_cfg.s.inst_hdr;
> +		if (port_cfg.s.hg_qos)
> +			pki_prt_cfg.style_cfg.parm_cfg.qpg_qos =
> CVMX_PKI_QPG_QOS_HIGIG;
> +		else if (port_cfg.s.qos_vlan)
> +			pki_prt_cfg.style_cfg.parm_cfg.qpg_qos =
> CVMX_PKI_QPG_QOS_VLAN;
> +		else if (port_cfg.s.qos_diff)
> +			pki_prt_cfg.style_cfg.parm_cfg.qpg_qos =
> CVMX_PKI_QPG_QOS_DIFFSERV;
> +		if (port_cfg.s.qos_vod)
> +			debug("Warning: 78xx needs pcam entries
> installed to achieve qos_vod\n");
> +		if (port_cfg.s.qos) {
> +			cvmx_pki_read_qpg_entry(xp.node,
> pki_prt_cfg.style_cfg.parm_cfg.qpg_base,
> +						&qpg_cfg);
> +			qpg_cfg.qpg_base = CVMX_PKI_FIND_AVAL_ENTRY;
> +			qpg_cfg.grp_ok |= port_cfg.s.qos;
> +			qpg_cfg.grp_bad |= port_cfg.s.qos;
> +			qpg_offset =
> cvmx_helper_pki_set_qpg_entry(xp.node, &qpg_cfg);
> +			if (qpg_offset == -1)
> +				debug("Warning: no new qpg entry
> available, will not modify qos\n");
> +			else
> +				pki_prt_cfg.style_cfg.parm_cfg.qpg_base
> = qpg_offset;
> +		}
> +		if (port_tag_cfg.s.grp !=
> pki_dflt_sso_grp[xp.node].group) {
> +			cvmx_pki_read_qpg_entry(xp.node,
> pki_prt_cfg.style_cfg.parm_cfg.qpg_base,
> +						&qpg_cfg);
> +			qpg_cfg.qpg_base = CVMX_PKI_FIND_AVAL_ENTRY;
> +			qpg_cfg.grp_ok |= (u8)(port_tag_cfg.s.grp <<
> 3);
> +			qpg_cfg.grp_bad |= (u8)(port_tag_cfg.s.grp <<
> 3);
> +			qpg_offset =
> cvmx_helper_pki_set_qpg_entry(xp.node, &qpg_cfg);
> +			if (qpg_offset == -1)
> +				debug("Warning: no new qpg entry
> available, will not modify group\n");
> +			else
> +				pki_prt_cfg.style_cfg.parm_cfg.qpg_base
> = qpg_offset;
> +		}
> +		pki_prt_cfg.pkind_cfg.parse_en.dsa_en =
> port_cfg.s.dsa_en;
> +		pki_prt_cfg.pkind_cfg.parse_en.hg_en =
> port_cfg.s.higig_en;
> +		pki_prt_cfg.style_cfg.tag_cfg.tag_fields.layer_c_src =
> +			port_tag_cfg.s.ip6_src_flag |
> port_tag_cfg.s.ip4_src_flag;
> +		pki_prt_cfg.style_cfg.tag_cfg.tag_fields.layer_c_dst =
> +			port_tag_cfg.s.ip6_dst_flag |
> port_tag_cfg.s.ip4_dst_flag;
> +		pki_prt_cfg.style_cfg.tag_cfg.tag_fields.ip_prot_nexthd
> r =
> +			port_tag_cfg.s.ip6_nxth_flag |
> port_tag_cfg.s.ip4_pctl_flag;
> +		pki_prt_cfg.style_cfg.tag_cfg.tag_fields.layer_d_src =
> +			port_tag_cfg.s.ip6_sprt_flag |
> port_tag_cfg.s.ip4_sprt_flag;
> +		pki_prt_cfg.style_cfg.tag_cfg.tag_fields.layer_d_dst =
> +			port_tag_cfg.s.ip6_dprt_flag |
> port_tag_cfg.s.ip4_dprt_flag;
> +		pki_prt_cfg.style_cfg.tag_cfg.tag_fields.input_port =
> port_tag_cfg.s.inc_prt_flag;
> +		pki_prt_cfg.style_cfg.tag_cfg.tag_fields.first_vlan =
> port_tag_cfg.s.inc_vlan;
> +		pki_prt_cfg.style_cfg.tag_cfg.tag_fields.second_vlan =
> port_tag_cfg.s.inc_vs;
> +
> +		if (port_tag_cfg.s.tcp6_tag_type ==
> port_tag_cfg.s.tcp4_tag_type)
> +			tcp_tag = port_tag_cfg.s.tcp6_tag_type;
> +		if (port_tag_cfg.s.ip6_tag_type ==
> port_tag_cfg.s.ip4_tag_type)
> +			ip_tag = port_tag_cfg.s.ip6_tag_type;
> +		pki_prt_cfg.style_cfg.parm_cfg.tag_type =
> +			(enum
> cvmx_sso_tag_type)port_tag_cfg.s.non_tag_type;
> +		if (tcp_tag == ip_tag && tcp_tag ==
> port_tag_cfg.s.non_tag_type)
> +			pki_prt_cfg.style_cfg.parm_cfg.tag_type = (enum
> cvmx_sso_tag_type)tcp_tag;
> +		else if (tcp_tag == ip_tag) {
> +			/* allocate and copy style */
> +			/* modify tag type */
> +			/*pcam entry for ip6 && ip4 match*/
> +			/* default is non tag type */
> +			__cvmx_pip_set_tag_type(xp.node, style, ip_tag,
> CVMX_PKI_PCAM_MATCH_IP);
> +		} else if (ip_tag == port_tag_cfg.s.non_tag_type) {
> +			/* allocate and copy style */
> +			/* modify tag type */
> +			/*pcam entry for tcp6 & tcp4 match*/
> +			/* default is non tag type */
> +			__cvmx_pip_set_tag_type(xp.node, style,
> tcp_tag, CVMX_PKI_PCAM_MATCH_TCP);
> +		} else {
> +			if (ip_tag != 0xaa) {
> +				nstyle =
> __cvmx_pip_set_tag_type(xp.node, style, ip_tag,
> +								 CVMX_P
> KI_PCAM_MATCH_IP);
> +				if (tcp_tag != 0xff)
> +					__cvmx_pip_set_tag_type(xp.node
> , nstyle, tcp_tag,
> +								CVMX_PK
> I_PCAM_MATCH_TCP);
> +				else {
> +					n4style =
> __cvmx_pip_set_tag_type(xp.node, nstyle, ip_tag,
> +									
>   CVMX_PKI_PCAM_MATCH_IPV4);
> +					__cvmx_pip_set_tag_type(xp.node
> , n4style,
> +								port_ta
> g_cfg.s.tcp4_tag_type,
> +								CVMX_PK
> I_PCAM_MATCH_TCP);
> +					n6style =
> __cvmx_pip_set_tag_type(xp.node, nstyle, ip_tag,
> +									
>   CVMX_PKI_PCAM_MATCH_IPV6);
> +					__cvmx_pip_set_tag_type(xp.node
> , n6style,
> +								port_ta
> g_cfg.s.tcp6_tag_type,
> +								CVMX_PK
> I_PCAM_MATCH_TCP);
> +				}
> +			} else {
> +				n4style =
> __cvmx_pip_set_tag_type(xp.node, style,
> +								  port_
> tag_cfg.s.ip4_tag_type,
> +								  CVMX_
> PKI_PCAM_MATCH_IPV4);
> +				n6style =
> __cvmx_pip_set_tag_type(xp.node, style,
> +								  port_
> tag_cfg.s.ip6_tag_type,
> +								  CVMX_
> PKI_PCAM_MATCH_IPV6);
> +				if (tcp_tag != 0xff) {
> +					__cvmx_pip_set_tag_type(xp.node
> , n4style, tcp_tag,
> +								CVMX_PK
> I_PCAM_MATCH_TCP);
> +					__cvmx_pip_set_tag_type(xp.node
> , n6style, tcp_tag,
> +								CVMX_PK
> I_PCAM_MATCH_TCP);
> +				} else {
> +					__cvmx_pip_set_tag_type(xp.node
> , n4style,
> +								port_ta
> g_cfg.s.tcp4_tag_type,
> +								CVMX_PK
> I_PCAM_MATCH_TCP);
> +					__cvmx_pip_set_tag_type(xp.node
> , n6style,
> +								port_ta
> g_cfg.s.tcp6_tag_type,
> +								CVMX_PK
> I_PCAM_MATCH_TCP);
> +				}
> +			}
> +		}
> +		pki_prt_cfg.style_cfg.parm_cfg.qpg_dis_padd =
> !port_tag_cfg.s.portadd_en;
> +
> +		if (port_cfg.s.mode == 0x1)
> +			pki_prt_cfg.pkind_cfg.initial_parse_mode =
> CVMX_PKI_PARSE_LA_TO_LG;
> +		else if (port_cfg.s.mode == 0x2)
> +			pki_prt_cfg.pkind_cfg.initial_parse_mode =
> CVMX_PKI_PARSE_LC_TO_LG;
> +		else
> +			pki_prt_cfg.pkind_cfg.initial_parse_mode =
> CVMX_PKI_PARSE_NOTHING;
> +		/* This is only for backward compatibility, not all the
> parameters are supported in 78xx */
> +		cvmx_pki_set_port_config(ipd_port, &pki_prt_cfg);
> +	} else {
> +		if (octeon_has_feature(OCTEON_FEATURE_PKND)) {
> +			int interface, index, pknd;
> +
> +			interface =
> cvmx_helper_get_interface_num(ipd_port);
> +			index =
> cvmx_helper_get_interface_index_num(ipd_port);
> +			pknd = cvmx_helper_get_pknd(interface, index);
> +
> +			ipd_port = pknd; /* overload port_num with pknd
> */
> +		}
> +		csr_wr(CVMX_PIP_PRT_CFGX(ipd_port), port_cfg.u64);
> +		csr_wr(CVMX_PIP_PRT_TAGX(ipd_port), port_tag_cfg.u64);
> +	}
> +}
> +
> +/**
> + * Configure the VLAN priority to QoS queue mapping.
> + *
> + * @param vlan_priority
> + *               VLAN priority (0-7)
> + * @param qos    QoS queue for packets matching this watcher
> + */
> +static inline void cvmx_pip_config_vlan_qos(u64 vlan_priority, u64
> qos)
> +{
> +	if (!octeon_has_feature(OCTEON_FEATURE_PKND)) {
> +		cvmx_pip_qos_vlanx_t pip_qos_vlanx;
> +
> +		pip_qos_vlanx.u64 = 0;
> +		pip_qos_vlanx.s.qos = qos;
> +		csr_wr(CVMX_PIP_QOS_VLANX(vlan_priority),
> pip_qos_vlanx.u64);
> +	}
> +}
> +
> +/**
> + * Configure the Diffserv to QoS queue mapping.
> + *
> + * @param diffserv Diffserv field value (0-63)
> + * @param qos      QoS queue for packets matching this watcher
> + */
> +static inline void cvmx_pip_config_diffserv_qos(u64 diffserv, u64
> qos)
> +{
> +	if (!octeon_has_feature(OCTEON_FEATURE_PKND)) {
> +		cvmx_pip_qos_diffx_t pip_qos_diffx;
> +
> +		pip_qos_diffx.u64 = 0;
> +		pip_qos_diffx.s.qos = qos;
> +		csr_wr(CVMX_PIP_QOS_DIFFX(diffserv),
> pip_qos_diffx.u64);
> +	}
> +}
> +
> +/**
> + * Get the status counters for a port for older non PKI chips.
> + *
> + * @param port_num Port number (ipd_port) to get statistics for.
> + * @param clear    Set to 1 to clear the counters after they are
> read
> + * @param status   Where to put the results.
> + */
> +static inline void cvmx_pip_get_port_stats(u64 port_num, u64 clear,
> cvmx_pip_port_status_t *status)
> +{
> +	cvmx_pip_stat_ctl_t pip_stat_ctl;
> +	cvmx_pip_stat0_prtx_t stat0;
> +	cvmx_pip_stat1_prtx_t stat1;
> +	cvmx_pip_stat2_prtx_t stat2;
> +	cvmx_pip_stat3_prtx_t stat3;
> +	cvmx_pip_stat4_prtx_t stat4;
> +	cvmx_pip_stat5_prtx_t stat5;
> +	cvmx_pip_stat6_prtx_t stat6;
> +	cvmx_pip_stat7_prtx_t stat7;
> +	cvmx_pip_stat8_prtx_t stat8;
> +	cvmx_pip_stat9_prtx_t stat9;
> +	cvmx_pip_stat10_x_t stat10;
> +	cvmx_pip_stat11_x_t stat11;
> +	cvmx_pip_stat_inb_pktsx_t pip_stat_inb_pktsx;
> +	cvmx_pip_stat_inb_octsx_t pip_stat_inb_octsx;
> +	cvmx_pip_stat_inb_errsx_t pip_stat_inb_errsx;
> +	int interface = cvmx_helper_get_interface_num(port_num);
> +	int index = cvmx_helper_get_interface_index_num(port_num);
> +
> +	pip_stat_ctl.u64 = 0;
> +	pip_stat_ctl.s.rdclr = clear;
> +	csr_wr(CVMX_PIP_STAT_CTL, pip_stat_ctl.u64);
> +
> +	if (octeon_has_feature(OCTEON_FEATURE_PKND)) {
> +		int pknd = cvmx_helper_get_pknd(interface, index);
> +		/*
> +		 * PIP_STAT_CTL[MODE] 0 means pkind.
> +		 */
> +		stat0.u64 = csr_rd(CVMX_PIP_STAT0_X(pknd));
> +		stat1.u64 = csr_rd(CVMX_PIP_STAT1_X(pknd));
> +		stat2.u64 = csr_rd(CVMX_PIP_STAT2_X(pknd));
> +		stat3.u64 = csr_rd(CVMX_PIP_STAT3_X(pknd));
> +		stat4.u64 = csr_rd(CVMX_PIP_STAT4_X(pknd));
> +		stat5.u64 = csr_rd(CVMX_PIP_STAT5_X(pknd));
> +		stat6.u64 = csr_rd(CVMX_PIP_STAT6_X(pknd));
> +		stat7.u64 = csr_rd(CVMX_PIP_STAT7_X(pknd));
> +		stat8.u64 = csr_rd(CVMX_PIP_STAT8_X(pknd));
> +		stat9.u64 = csr_rd(CVMX_PIP_STAT9_X(pknd));
> +		stat10.u64 = csr_rd(CVMX_PIP_STAT10_X(pknd));
> +		stat11.u64 = csr_rd(CVMX_PIP_STAT11_X(pknd));
> +	} else {
> +		if (port_num >= 40) {
> +			stat0.u64 =
> csr_rd(CVMX_PIP_XSTAT0_PRTX(port_num));
> +			stat1.u64 =
> csr_rd(CVMX_PIP_XSTAT1_PRTX(port_num));
> +			stat2.u64 =
> csr_rd(CVMX_PIP_XSTAT2_PRTX(port_num));
> +			stat3.u64 =
> csr_rd(CVMX_PIP_XSTAT3_PRTX(port_num));
> +			stat4.u64 =
> csr_rd(CVMX_PIP_XSTAT4_PRTX(port_num));
> +			stat5.u64 =
> csr_rd(CVMX_PIP_XSTAT5_PRTX(port_num));
> +			stat6.u64 =
> csr_rd(CVMX_PIP_XSTAT6_PRTX(port_num));
> +			stat7.u64 =
> csr_rd(CVMX_PIP_XSTAT7_PRTX(port_num));
> +			stat8.u64 =
> csr_rd(CVMX_PIP_XSTAT8_PRTX(port_num));
> +			stat9.u64 =
> csr_rd(CVMX_PIP_XSTAT9_PRTX(port_num));
> +			if (OCTEON_IS_MODEL(OCTEON_CN6XXX)) {
> +				stat10.u64 =
> csr_rd(CVMX_PIP_XSTAT10_PRTX(port_num));
> +				stat11.u64 =
> csr_rd(CVMX_PIP_XSTAT11_PRTX(port_num));
> +			}
> +		} else {
> +			stat0.u64 =
> csr_rd(CVMX_PIP_STAT0_PRTX(port_num));
> +			stat1.u64 =
> csr_rd(CVMX_PIP_STAT1_PRTX(port_num));
> +			stat2.u64 =
> csr_rd(CVMX_PIP_STAT2_PRTX(port_num));
> +			stat3.u64 =
> csr_rd(CVMX_PIP_STAT3_PRTX(port_num));
> +			stat4.u64 =
> csr_rd(CVMX_PIP_STAT4_PRTX(port_num));
> +			stat5.u64 =
> csr_rd(CVMX_PIP_STAT5_PRTX(port_num));
> +			stat6.u64 =
> csr_rd(CVMX_PIP_STAT6_PRTX(port_num));
> +			stat7.u64 =
> csr_rd(CVMX_PIP_STAT7_PRTX(port_num));
> +			stat8.u64 =
> csr_rd(CVMX_PIP_STAT8_PRTX(port_num));
> +			stat9.u64 =
> csr_rd(CVMX_PIP_STAT9_PRTX(port_num));
> +			if (OCTEON_IS_OCTEON2() ||
> OCTEON_IS_MODEL(OCTEON_CN70XX)) {
> +				stat10.u64 =
> csr_rd(CVMX_PIP_STAT10_PRTX(port_num));
> +				stat11.u64 =
> csr_rd(CVMX_PIP_STAT11_PRTX(port_num));
> +			}
> +		}
> +	}
> +	if (octeon_has_feature(OCTEON_FEATURE_PKND)) {
> +		int pknd = cvmx_helper_get_pknd(interface, index);
> +
> +		pip_stat_inb_pktsx.u64 =
> csr_rd(CVMX_PIP_STAT_INB_PKTS_PKNDX(pknd));
> +		pip_stat_inb_octsx.u64 =
> csr_rd(CVMX_PIP_STAT_INB_OCTS_PKNDX(pknd));
> +		pip_stat_inb_errsx.u64 =
> csr_rd(CVMX_PIP_STAT_INB_ERRS_PKNDX(pknd));
> +	} else {
> +		pip_stat_inb_pktsx.u64 =
> csr_rd(CVMX_PIP_STAT_INB_PKTSX(port_num));
> +		pip_stat_inb_octsx.u64 =
> csr_rd(CVMX_PIP_STAT_INB_OCTSX(port_num));
> +		pip_stat_inb_errsx.u64 =
> csr_rd(CVMX_PIP_STAT_INB_ERRSX(port_num));
> +	}
> +
> +	status->dropped_octets = stat0.s.drp_octs;
> +	status->dropped_packets = stat0.s.drp_pkts;
> +	status->octets = stat1.s.octs;
> +	status->pci_raw_packets = stat2.s.raw;
> +	status->packets = stat2.s.pkts;
> +	status->multicast_packets = stat3.s.mcst;
> +	status->broadcast_packets = stat3.s.bcst;
> +	status->len_64_packets = stat4.s.h64;
> +	status->len_65_127_packets = stat4.s.h65to127;
> +	status->len_128_255_packets = stat5.s.h128to255;
> +	status->len_256_511_packets = stat5.s.h256to511;
> +	status->len_512_1023_packets = stat6.s.h512to1023;
> +	status->len_1024_1518_packets = stat6.s.h1024to1518;
> +	status->len_1519_max_packets = stat7.s.h1519;
> +	status->fcs_align_err_packets = stat7.s.fcs;
> +	status->runt_packets = stat8.s.undersz;
> +	status->runt_crc_packets = stat8.s.frag;
> +	status->oversize_packets = stat9.s.oversz;
> +	status->oversize_crc_packets = stat9.s.jabber;
> +	if (OCTEON_IS_OCTEON2() || OCTEON_IS_MODEL(OCTEON_CN70XX)) {
> +		status->mcast_l2_red_packets = stat10.s.mcast;
> +		status->bcast_l2_red_packets = stat10.s.bcast;
> +		status->mcast_l3_red_packets = stat11.s.mcast;
> +		status->bcast_l3_red_packets = stat11.s.bcast;
> +	}
> +	status->inb_packets = pip_stat_inb_pktsx.s.pkts;
> +	status->inb_octets = pip_stat_inb_octsx.s.octs;
> +	status->inb_errors = pip_stat_inb_errsx.s.errs;
> +}
> +
> +/**
> + * Get the status counters for a port.
> + *
> + * @param port_num Port number (ipd_port) to get statistics for.
> + * @param clear    Set to 1 to clear the counters after they are
> read
> + * @param status   Where to put the results.
> + */
> +static inline void cvmx_pip_get_port_status(u64 port_num, u64 clear,
> cvmx_pip_port_status_t *status)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_PKI)) {
> +		unsigned int node = cvmx_get_node_num();
> +
> +		cvmx_pki_get_port_stats(node, port_num, (struct
> cvmx_pki_port_stats *)status);
> +	} else {
> +		cvmx_pip_get_port_stats(port_num, clear, status);
> +	}
> +}
> +
> +/**
> + * Configure the hardware CRC engine
> + *
> + * @param interface Interface to configure (0 or 1)
> + * @param invert_result
> + *                 Invert the result of the CRC
> + * @param reflect  Reflect
> + * @param initialization_vector
> + *                 CRC initialization vector
> + */
> +static inline void cvmx_pip_config_crc(u64 interface, u64
> invert_result, u64 reflect,
> +				       u32 initialization_vector)
> +{
> +	/* Only CN38XX & CN58XX */
> +}
> +
> +/**
> + * Clear all bits in a tag mask. This should be called on
> + * startup before any calls to cvmx_pip_tag_mask_set. Each bit
> + * set in the final mask represent a byte used in the packet for
> + * tag generation.
> + *
> + * @param mask_index Which tag mask to clear (0..3)
> + */
> +static inline void cvmx_pip_tag_mask_clear(u64 mask_index)
> +{
> +	u64 index;
> +	cvmx_pip_tag_incx_t pip_tag_incx;
> +
> +	pip_tag_incx.u64 = 0;
> +	pip_tag_incx.s.en = 0;
> +	for (index = mask_index * 16; index < (mask_index + 1) * 16;
> index++)
> +		csr_wr(CVMX_PIP_TAG_INCX(index), pip_tag_incx.u64);
> +}
> +
> +/**
> + * Sets a range of bits in the tag mask. The tag mask is used
> + * when the cvmx_pip_port_tag_cfg_t tag_mode is non zero.
> + * There are four separate masks that can be configured.
> + *
> + * @param mask_index Which tag mask to modify (0..3)
> + * @param offset     Offset into the bitmask to set bits at. Use the
> GCC macro
> + *                   offsetof() to determine the offsets into packet
> headers.
> + *                   For example, offsetof(ethhdr, protocol) returns
> the offset
> + *                   of the ethernet protocol field.  The bitmask
> selects which bytes
> + *                   to include the the tag, with bit offset X
> selecting byte at offset X
> + *                   from the beginning of the packet data.
> + * @param len        Number of bytes to include. Usually this is the
> sizeof()
> + *                   the field.
> + */
> +static inline void cvmx_pip_tag_mask_set(u64 mask_index, u64 offset,
> u64 len)
> +{
> +	while (len--) {
> +		cvmx_pip_tag_incx_t pip_tag_incx;
> +		u64 index = mask_index * 16 + offset / 8;
> +
> +		pip_tag_incx.u64 = csr_rd(CVMX_PIP_TAG_INCX(index));
> +		pip_tag_incx.s.en |= 0x80 >> (offset & 0x7);
> +		csr_wr(CVMX_PIP_TAG_INCX(index), pip_tag_incx.u64);
> +		offset++;
> +	}
> +}
> +
> +/**
> + * Set byte count for Max-Sized and Min Sized frame check.
> + *
> + * @param interface   Which interface to set the limit
> + * @param max_size    Byte count for Max-Size frame check
> + */
> +static inline void cvmx_pip_set_frame_check(int interface, u32
> max_size)
> +{
> +	cvmx_pip_frm_len_chkx_t frm_len;
> +
> +	/* max_size and min_size are passed as 0, reset to default
> values. */
> +	if (max_size < 1536)
> +		max_size = 1536;
> +
> +	/* On CN68XX frame check is enabled for a pkind n and
> +	   PIP_PRT_CFG[len_chk_sel] selects which set of
> +	   MAXLEN/MINLEN to use. */
> +	if (octeon_has_feature(OCTEON_FEATURE_PKND)) {
> +		int port;
> +		int num_ports =
> cvmx_helper_ports_on_interface(interface);
> +
> +		for (port = 0; port < num_ports; port++) {
> +			if (octeon_has_feature(OCTEON_FEATURE_PKI)) {
> +				int ipd_port;
> +
> +				ipd_port =
> cvmx_helper_get_ipd_port(interface, port);
> +				cvmx_pki_set_max_frm_len(ipd_port,
> max_size);
> +			} else {
> +				int pknd;
> +				int sel;
> +				cvmx_pip_prt_cfgx_t config;
> +
> +				pknd = cvmx_helper_get_pknd(interface,
> port);
> +				config.u64 =
> csr_rd(CVMX_PIP_PRT_CFGX(pknd));
> +				sel = config.s.len_chk_sel;
> +				frm_len.u64 =
> csr_rd(CVMX_PIP_FRM_LEN_CHKX(sel));
> +				frm_len.s.maxlen = max_size;
> +				csr_wr(CVMX_PIP_FRM_LEN_CHKX(sel),
> frm_len.u64);
> +			}
> +		}
> +	}
> +	/* on cn6xxx and cn7xxx models, PIP_FRM_LEN_CHK0 applies to
> +	 *     all incoming traffic */
> +	else if (OCTEON_IS_OCTEON2() || OCTEON_IS_MODEL(OCTEON_CN70XX))
> {
> +		frm_len.u64 = csr_rd(CVMX_PIP_FRM_LEN_CHKX(0));
> +		frm_len.s.maxlen = max_size;
> +		csr_wr(CVMX_PIP_FRM_LEN_CHKX(0), frm_len.u64);
> +	}
> +}
> +
> +/**
> + * Initialize Bit Select Extractor config. Their are 8 bit positions
> and valids
> + * to be used when using the corresponding extractor.
> + *
> + * @param bit     Bit Select Extractor to use
> + * @param pos     Which position to update
> + * @param val     The value to update the position with
> + */
> +static inline void cvmx_pip_set_bsel_pos(int bit, int pos, int val)
> +{
> +	cvmx_pip_bsel_ext_posx_t bsel_pos;
> +
> +	/* The bit select extractor is available in CN61XX and CN68XX
> pass2.0 onwards. */
> +	if (!octeon_has_feature(OCTEON_FEATURE_BIT_EXTRACTOR))
> +		return;
> +
> +	if (bit < 0 || bit > 3) {
> +		debug("ERROR: cvmx_pip_set_bsel_pos: Invalid Bit-Select 
> Extractor (%d) passed\n",
> +		      bit);
> +		return;
> +	}
> +
> +	bsel_pos.u64 = csr_rd(CVMX_PIP_BSEL_EXT_POSX(bit));
> +	switch (pos) {
> +	case 0:
> +		bsel_pos.s.pos0_val = 1;
> +		bsel_pos.s.pos0 = val & 0x7f;
> +		break;
> +	case 1:
> +		bsel_pos.s.pos1_val = 1;
> +		bsel_pos.s.pos1 = val & 0x7f;
> +		break;
> +	case 2:
> +		bsel_pos.s.pos2_val = 1;
> +		bsel_pos.s.pos2 = val & 0x7f;
> +		break;
> +	case 3:
> +		bsel_pos.s.pos3_val = 1;
> +		bsel_pos.s.pos3 = val & 0x7f;
> +		break;
> +	case 4:
> +		bsel_pos.s.pos4_val = 1;
> +		bsel_pos.s.pos4 = val & 0x7f;
> +		break;
> +	case 5:
> +		bsel_pos.s.pos5_val = 1;
> +		bsel_pos.s.pos5 = val & 0x7f;
> +		break;
> +	case 6:
> +		bsel_pos.s.pos6_val = 1;
> +		bsel_pos.s.pos6 = val & 0x7f;
> +		break;
> +	case 7:
> +		bsel_pos.s.pos7_val = 1;
> +		bsel_pos.s.pos7 = val & 0x7f;
> +		break;
> +	default:
> +		debug("Warning: cvmx_pip_set_bsel_pos: Invalid
> pos(%d)\n", pos);
> +		break;
> +	}
> +	csr_wr(CVMX_PIP_BSEL_EXT_POSX(bit), bsel_pos.u64);
> +}
> +
> +/**
> + * Initialize offset and skip values to use by bit select extractor.
> +
> + * @param bit	Bit Select Extractor to use
> + * @param offset	Offset to add to extractor mem addr to get
> final address
> + *			to lookup table.
> + * @param skip		Number of bytes to skip from start of
> packet 0-64
> + */
> +static inline void cvmx_pip_bsel_config(int bit, int offset, int
> skip)
> +{
> +	cvmx_pip_bsel_ext_cfgx_t bsel_cfg;
> +
> +	/* The bit select extractor is available in CN61XX and CN68XX
> pass2.0 onwards. */
> +	if (!octeon_has_feature(OCTEON_FEATURE_BIT_EXTRACTOR))
> +		return;
> +
> +	bsel_cfg.u64 = csr_rd(CVMX_PIP_BSEL_EXT_CFGX(bit));
> +	bsel_cfg.s.offset = offset;
> +	bsel_cfg.s.skip = skip;
> +	csr_wr(CVMX_PIP_BSEL_EXT_CFGX(bit), bsel_cfg.u64);
> +}
> +
> +/**
> + * Get the entry for the Bit Select Extractor Table.
> + * @param work   pointer to work queue entry
> + * @return       Index of the Bit Select Extractor Table
> + */
> +static inline int cvmx_pip_get_bsel_table_index(cvmx_wqe_t *work)
> +{
> +	int bit = cvmx_wqe_get_port(work) & 0x3;
> +	/* Get the Bit select table index. */
> +	int index;
> +	int y;
> +	cvmx_pip_bsel_ext_cfgx_t bsel_cfg;
> +	cvmx_pip_bsel_ext_posx_t bsel_pos;
> +
> +	/* The bit select extractor is available in CN61XX and CN68XX
> pass2.0 onwards. */
> +	if (!octeon_has_feature(OCTEON_FEATURE_BIT_EXTRACTOR))
> +		return -1;
> +
> +	bsel_cfg.u64 = csr_rd(CVMX_PIP_BSEL_EXT_CFGX(bit));
> +	bsel_pos.u64 = csr_rd(CVMX_PIP_BSEL_EXT_POSX(bit));
> +
> +	for (y = 0; y < 8; y++) {
> +		char *ptr = (char *)cvmx_phys_to_ptr(work-
> >packet_ptr.s.addr);
> +		int bit_loc = 0;
> +		int bit;
> +
> +		ptr += bsel_cfg.s.skip;
> +		switch (y) {
> +		case 0:
> +			ptr += (bsel_pos.s.pos0 >> 3);
> +			bit_loc = 7 - (bsel_pos.s.pos0 & 0x3);
> +			break;
> +		case 1:
> +			ptr += (bsel_pos.s.pos1 >> 3);
> +			bit_loc = 7 - (bsel_pos.s.pos1 & 0x3);
> +			break;
> +		case 2:
> +			ptr += (bsel_pos.s.pos2 >> 3);
> +			bit_loc = 7 - (bsel_pos.s.pos2 & 0x3);
> +			break;
> +		case 3:
> +			ptr += (bsel_pos.s.pos3 >> 3);
> +			bit_loc = 7 - (bsel_pos.s.pos3 & 0x3);
> +			break;
> +		case 4:
> +			ptr += (bsel_pos.s.pos4 >> 3);
> +			bit_loc = 7 - (bsel_pos.s.pos4 & 0x3);
> +			break;
> +		case 5:
> +			ptr += (bsel_pos.s.pos5 >> 3);
> +			bit_loc = 7 - (bsel_pos.s.pos5 & 0x3);
> +			break;
> +		case 6:
> +			ptr += (bsel_pos.s.pos6 >> 3);
> +			bit_loc = 7 - (bsel_pos.s.pos6 & 0x3);
> +			break;
> +		case 7:
> +			ptr += (bsel_pos.s.pos7 >> 3);
> +			bit_loc = 7 - (bsel_pos.s.pos7 & 0x3);
> +			break;
> +		}
> +		bit = (*ptr >> bit_loc) & 1;
> +		index |= bit << y;
> +	}
> +	index += bsel_cfg.s.offset;
> +	index &= 0x1ff;
> +	return index;
> +}
> +
> +static inline int cvmx_pip_get_bsel_qos(cvmx_wqe_t *work)
> +{
> +	int index = cvmx_pip_get_bsel_table_index(work);
> +	cvmx_pip_bsel_tbl_entx_t bsel_tbl;
> +
> +	/* The bit select extractor is available in CN61XX and CN68XX
> pass2.0 onwards. */
> +	if (!octeon_has_feature(OCTEON_FEATURE_BIT_EXTRACTOR))
> +		return -1;
> +
> +	bsel_tbl.u64 = csr_rd(CVMX_PIP_BSEL_TBL_ENTX(index));
> +
> +	return bsel_tbl.s.qos;
> +}
> +
> +static inline int cvmx_pip_get_bsel_grp(cvmx_wqe_t *work)
> +{
> +	int index = cvmx_pip_get_bsel_table_index(work);
> +	cvmx_pip_bsel_tbl_entx_t bsel_tbl;
> +
> +	/* The bit select extractor is available in CN61XX and CN68XX
> pass2.0 onwards. */
> +	if (!octeon_has_feature(OCTEON_FEATURE_BIT_EXTRACTOR))
> +		return -1;
> +
> +	bsel_tbl.u64 = csr_rd(CVMX_PIP_BSEL_TBL_ENTX(index));
> +
> +	return bsel_tbl.s.grp;
> +}
> +
> +static inline int cvmx_pip_get_bsel_tt(cvmx_wqe_t *work)
> +{
> +	int index = cvmx_pip_get_bsel_table_index(work);
> +	cvmx_pip_bsel_tbl_entx_t bsel_tbl;
> +
> +	/* The bit select extractor is available in CN61XX and CN68XX
> pass2.0 onwards. */
> +	if (!octeon_has_feature(OCTEON_FEATURE_BIT_EXTRACTOR))
> +		return -1;
> +
> +	bsel_tbl.u64 = csr_rd(CVMX_PIP_BSEL_TBL_ENTX(index));
> +
> +	return bsel_tbl.s.tt;
> +}
> +
> +static inline int cvmx_pip_get_bsel_tag(cvmx_wqe_t *work)
> +{
> +	int index = cvmx_pip_get_bsel_table_index(work);
> +	int port = cvmx_wqe_get_port(work);
> +	int bit = port & 0x3;
> +	int upper_tag = 0;
> +	cvmx_pip_bsel_tbl_entx_t bsel_tbl;
> +	cvmx_pip_bsel_ext_cfgx_t bsel_cfg;
> +	cvmx_pip_prt_tagx_t prt_tag;
> +
> +	/* The bit select extractor is available in CN61XX and CN68XX
> pass2.0 onwards. */
> +	if (!octeon_has_feature(OCTEON_FEATURE_BIT_EXTRACTOR))
> +		return -1;
> +
> +	bsel_tbl.u64 = csr_rd(CVMX_PIP_BSEL_TBL_ENTX(index));
> +	bsel_cfg.u64 = csr_rd(CVMX_PIP_BSEL_EXT_CFGX(bit));
> +
> +	prt_tag.u64 = csr_rd(CVMX_PIP_PRT_TAGX(port));
> +	if (prt_tag.s.inc_prt_flag == 0)
> +		upper_tag = bsel_cfg.s.upper_tag;
> +	return bsel_tbl.s.tag | ((bsel_cfg.s.tag << 8) & 0xff00) |
> ((upper_tag << 16) & 0xffff0000);
> +}
> +
> +#endif /*  __CVMX_PIP_H__ */
> diff --git a/arch/mips/mach-octeon/include/mach/cvmx-pki-resources.h
> b/arch/mips/mach-octeon/include/mach/cvmx-pki-resources.h
> new file mode 100644
> index 000000000000..79b99b0bd7c2
> --- /dev/null
> +++ b/arch/mips/mach-octeon/include/mach/cvmx-pki-resources.h
> @@ -0,0 +1,157 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +/*
> + * Copyright (C) 2020 Marvell International Ltd.
> + *
> + * Resource management for PKI resources.
> + */
> +
> +#ifndef __CVMX_PKI_RESOURCES_H__
> +#define __CVMX_PKI_RESOURCES_H__
> +
> +/**
> + * This function allocates/reserves a style from pool of global
> styles per node.
> + * @param node	 node to allocate style from.
> + * @param style	 style to allocate, if -1 it will be allocated
> +		 first available style from style resource. If index is
> positive
> +		 number and in range, it will try to allocate specified
> style.
> + * @return	 style number on success, -1 on failure.
> + */
> +int cvmx_pki_style_alloc(int node, int style);
> +
> +/**
> + * This function allocates/reserves a cluster group from per node
> +   cluster group resources.
> + * @param node		node to allocate cluster group from.
> +   @param cl_grp	cluster group to allocate/reserve, if -1 ,
> +			allocate any available cluster group.
> + * @return		cluster group number or -1 on failure
> + */
> +int cvmx_pki_cluster_grp_alloc(int node, int cl_grp);
> +
> +/**
> + * This function allocates/reserves a cluster from per node
> +   cluster resources.
> + * @param node		node to allocate cluster group from.
> +   @param cluster_mask	mask of clusters  to allocate/reserve,
> if -1 ,
> +			allocate any available clusters.
> + * @param num_clusters	number of clusters that will be
> allocated
> + */
> +int cvmx_pki_cluster_alloc(int node, int num_clusters, u64
> *cluster_mask);
> +
> +/**
> + * This function allocates/reserves a pcam entry from node
> + * @param node		node to allocate pcam entry from.
> +   @param index	index of pacm entry (0-191), if -1 ,
> +			allocate any available pcam entry.
> + * @param bank		pcam bank where to allocate/reserve
> pcan entry from
> + * @param cluster_mask  mask of clusters from which pcam entry is
> needed.
> + * @return		pcam entry of -1 on failure
> + */
> +int cvmx_pki_pcam_entry_alloc(int node, int index, int bank, u64
> cluster_mask);
> +
> +/**
> + * This function allocates/reserves QPG table entries per node.
> + * @param node		node number.
> + * @param base_offset	base_offset in qpg table. If -1, first
> available
> +			qpg base_offset will be allocated. If
> base_offset is positive
> +			number and in range, it will try to allocate
> specified base_offset.
> +   @param count		number of consecutive qpg entries to
> allocate. They will be consecutive
> +			from base offset.
> + * @return		qpg table base offset number on success, -1 on
> failure.
> + */
> +int cvmx_pki_qpg_entry_alloc(int node, int base_offset, int count);
> +
> +/**
> + * This function frees a style from pool of global styles per node.
> + * @param node	 node to free style from.
> + * @param style	 style to free
> + * @return	 0 on success, -1 on failure.
> + */
> +int cvmx_pki_style_free(int node, int style);
> +
> +/**
> + * This function frees a cluster group from per node
> +   cluster group resources.
> + * @param node		node to free cluster group from.
> +   @param cl_grp	cluster group to free
> + * @return		0 on success or -1 on failure
> + */
> +int cvmx_pki_cluster_grp_free(int node, int cl_grp);
> +
> +/**
> + * This function frees QPG table entries per node.
> + * @param node		node number.
> + * @param base_offset	base_offset in qpg table. If -1, first
> available
> + *			qpg base_offset will be allocated. If
> base_offset is positive
> + *			number and in range, it will try to allocate
> specified base_offset.
> + * @param count		number of consecutive qpg entries to
> allocate. They will be consecutive
> + *			from base offset.
> + * @return		qpg table base offset number on success, -1 on
> failure.
> + */
> +int cvmx_pki_qpg_entry_free(int node, int base_offset, int count);
> +
> +/**
> + * This function frees  clusters  from per node
> +   clusters resources.
> + * @param node		node to free clusters from.
> + * @param cluster_mask  mask of clusters need freeing
> + * @return		0 on success or -1 on failure
> + */
> +int cvmx_pki_cluster_free(int node, u64 cluster_mask);
> +
> +/**
> + * This function frees a pcam entry from node
> + * @param node		node to allocate pcam entry from.
> +   @param index	index of pacm entry (0-191) needs to be freed.
> + * @param bank		pcam bank where to free pcam entry from
> + * @param cluster_mask  mask of clusters from which pcam entry is
> freed.
> + * @return		0 on success OR -1 on failure
> + */
> +int cvmx_pki_pcam_entry_free(int node, int index, int bank, u64
> cluster_mask);
> +
> +/**
> + * This function allocates/reserves a bpid from pool of global bpid
> per node.
> + * @param node	node to allocate bpid from.
> + * @param bpid	bpid  to allocate, if -1 it will be allocated
> + *		first available boid from bpid resource. If index is
> positive
> + *		number and in range, it will try to allocate specified
> bpid.
> + * @return	bpid number on success,
> + *		-1 on alloc failure.
> + *		-2 on resource already reserved.
> + */
> +int cvmx_pki_bpid_alloc(int node, int bpid);
> +
> +/**
> + * This function frees a bpid from pool of global bpid per node.
> + * @param node	 node to free bpid from.
> + * @param bpid	 bpid to free
> + * @return	 0 on success, -1 on failure or
> + */
> +int cvmx_pki_bpid_free(int node, int bpid);
> +
> +/**
> + * This function frees all the PKI software resources
> + * (clusters, styles, qpg_entry, pcam_entry etc) for the specified
> node
> + */
> +
> +/**
> + * This function allocates/reserves an index from pool of global
> MTAG-IDX per node.
> + * @param node	node to allocate index from.
> + * @param idx	index  to allocate, if -1 it will be allocated
> + * @return	MTAG index number on success,
> + *		-1 on alloc failure.
> + *		-2 on resource already reserved.
> + */
> +int cvmx_pki_mtag_idx_alloc(int node, int idx);
> +
> +/**
> + * This function frees an index from pool of global MTAG-IDX per
> node.
> + * @param node	 node to free bpid from.
> + * @param bpid	 bpid to free
> + * @return	 0 on success, -1 on failure or
> + */
> +int cvmx_pki_mtag_idx_free(int node, int idx);
> +
> +void __cvmx_pki_global_rsrc_free(int node);
> +
> +#endif /*  __CVM_PKI_RESOURCES_H__ */
> diff --git a/arch/mips/mach-octeon/include/mach/cvmx-pki.h
> b/arch/mips/mach-octeon/include/mach/cvmx-pki.h
> new file mode 100644
> index 000000000000..c1feb55a1f01
> --- /dev/null
> +++ b/arch/mips/mach-octeon/include/mach/cvmx-pki.h
> @@ -0,0 +1,970 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +/*
> + * Copyright (C) 2020 Marvell International Ltd.
> + *
> + * Interface to the hardware Packet Input Data unit.
> + */
> +
> +#ifndef __CVMX_PKI_H__
> +#define __CVMX_PKI_H__
> +
> +#include "cvmx-fpa3.h"
> +#include "cvmx-helper-util.h"
> +#include "cvmx-helper-cfg.h"
> +#include "cvmx-error.h"
> +
> +/* PKI AURA and BPID count are equal to FPA AURA count */
> +#define CVMX_PKI_NUM_AURA	       (cvmx_fpa3_num_auras())
> +#define CVMX_PKI_NUM_BPID	       (cvmx_fpa3_num_auras())
> +#define CVMX_PKI_NUM_SSO_GROUP	       (cvmx_sso_num_xgrp())
> +#define CVMX_PKI_NUM_CLUSTER_GROUP_MAX 1
> +#define CVMX_PKI_NUM_CLUSTER_GROUP     (cvmx_pki_num_cl_grp())
> +#define CVMX_PKI_NUM_CLUSTER	       (cvmx_pki_num_clusters())
> +
> +/* FIXME: Reduce some of these values, convert to routines XXX */
> +#define CVMX_PKI_NUM_CHANNEL	    4096
> +#define CVMX_PKI_NUM_PKIND	    64
> +#define CVMX_PKI_NUM_INTERNAL_STYLE 256
> +#define CVMX_PKI_NUM_FINAL_STYLE    64
> +#define CVMX_PKI_NUM_QPG_ENTRY	    2048
> +#define CVMX_PKI_NUM_MTAG_IDX	    (32 / 4) /* 32 registers
> grouped by 4*/
> +#define CVMX_PKI_NUM_LTYPE	    32
> +#define CVMX_PKI_NUM_PCAM_BANK	    2
> +#define CVMX_PKI_NUM_PCAM_ENTRY	    192
> +#define CVMX_PKI_NUM_FRAME_CHECK    2
> +#define CVMX_PKI_NUM_BELTYPE	    32
> +#define CVMX_PKI_MAX_FRAME_SIZE	    65535
> +#define CVMX_PKI_FIND_AVAL_ENTRY    (-1)
> +#define CVMX_PKI_CLUSTER_ALL	    0xf
> +
> +#ifdef CVMX_SUPPORT_SEPARATE_CLUSTER_CONFIG
> +#define
> CVMX_PKI_TOTAL_PCAM_ENTRY                                            
>                       \
> +	((CVMX_PKI_NUM_CLUSTER) * (CVMX_PKI_NUM_PCAM_BANK) *
> (CVMX_PKI_NUM_PCAM_ENTRY))
> +#else
> +#define CVMX_PKI_TOTAL_PCAM_ENTRY (CVMX_PKI_NUM_PCAM_BANK *
> CVMX_PKI_NUM_PCAM_ENTRY)
> +#endif
> +
> +static inline unsigned int cvmx_pki_num_clusters(void)
> +{
> +	if (OCTEON_IS_MODEL(OCTEON_CN73XX) ||
> OCTEON_IS_MODEL(OCTEON_CNF75XX))
> +		return 2;
> +	return 4;
> +}
> +
> +static inline unsigned int cvmx_pki_num_cl_grp(void)
> +{
> +	if (OCTEON_IS_MODEL(OCTEON_CN73XX) ||
> OCTEON_IS_MODEL(OCTEON_CNF75XX) ||
> +	    OCTEON_IS_MODEL(OCTEON_CN78XX))
> +		return 1;
> +	return 0;
> +}
> +
> +enum cvmx_pki_pkind_parse_mode {
> +	CVMX_PKI_PARSE_LA_TO_LG = 0,  /* Parse LA(L2) to LG */
> +	CVMX_PKI_PARSE_LB_TO_LG = 1,  /* Parse LB(custom) to LG */
> +	CVMX_PKI_PARSE_LC_TO_LG = 3,  /* Parse LC(L3) to LG */
> +	CVMX_PKI_PARSE_LG = 0x3f,     /* Parse LG */
> +	CVMX_PKI_PARSE_NOTHING = 0x7f /* Parse nothing */
> +};
> +
> +enum cvmx_pki_parse_mode_chg {
> +	CVMX_PKI_PARSE_NO_CHG = 0x0,
> +	CVMX_PKI_PARSE_SKIP_TO_LB = 0x1,
> +	CVMX_PKI_PARSE_SKIP_TO_LC = 0x3,
> +	CVMX_PKI_PARSE_SKIP_TO_LD = 0x7,
> +	CVMX_PKI_PARSE_SKIP_TO_LG = 0x3f,
> +	CVMX_PKI_PARSE_SKIP_ALL = 0x7f,
> +};
> +
> +enum cvmx_pki_l2_len_mode { PKI_L2_LENCHK_EQUAL_GREATER = 0,
> PKI_L2_LENCHK_EQUAL_ONLY };
> +
> +enum cvmx_pki_cache_mode {
> +	CVMX_PKI_OPC_MODE_STT = 0LL,	  /* All blocks write through
> DRAM,*/
> +	CVMX_PKI_OPC_MODE_STF = 1LL,	  /* All blocks into L2 */
> +	CVMX_PKI_OPC_MODE_STF1_STT = 2LL, /* 1st block L2, rest DRAM */
> +	CVMX_PKI_OPC_MODE_STF2_STT = 3LL  /* 1st, 2nd blocks L2, rest
> DRAM */
> +};
> +
> +/**
> + * Tag type definitions
> + */
> +enum cvmx_sso_tag_type {
> +	CVMX_SSO_TAG_TYPE_ORDERED = 0L,
> +	CVMX_SSO_TAG_TYPE_ATOMIC = 1L,
> +	CVMX_SSO_TAG_TYPE_UNTAGGED = 2L,
> +	CVMX_SSO_TAG_TYPE_EMPTY = 3L
> +};
> +
> +enum cvmx_pki_qpg_qos {
> +	CVMX_PKI_QPG_QOS_NONE = 0,
> +	CVMX_PKI_QPG_QOS_VLAN,
> +	CVMX_PKI_QPG_QOS_MPLS,
> +	CVMX_PKI_QPG_QOS_DSA_SRC,
> +	CVMX_PKI_QPG_QOS_DIFFSERV,
> +	CVMX_PKI_QPG_QOS_HIGIG,
> +};
> +
> +enum cvmx_pki_wqe_vlan { CVMX_PKI_USE_FIRST_VLAN = 0,
> CVMX_PKI_USE_SECOND_VLAN };
> +
> +/**
> + * Controls how the PKI statistics counters are handled
> + * The PKI_STAT*_X registers can be indexed either by port kind
> (pkind), or
> + * final style. (Does not apply to the PKI_STAT_INB* registers.)
> + *    0 = X represents the packet’s pkind
> + *    1 = X represents the low 6-bits of packet’s final style
> + */
> +enum cvmx_pki_stats_mode { CVMX_PKI_STAT_MODE_PKIND,
> CVMX_PKI_STAT_MODE_STYLE };
> +
> +enum cvmx_pki_fpa_wait { CVMX_PKI_DROP_PKT, CVMX_PKI_WAIT_PKT };
> +
> +#define PKI_BELTYPE_E__NONE_M 0x0
> +#define PKI_BELTYPE_E__MISC_M 0x1
> +#define PKI_BELTYPE_E__IP4_M  0x2
> +#define PKI_BELTYPE_E__IP6_M  0x3
> +#define PKI_BELTYPE_E__TCP_M  0x4
> +#define PKI_BELTYPE_E__UDP_M  0x5
> +#define PKI_BELTYPE_E__SCTP_M 0x6
> +#define PKI_BELTYPE_E__SNAP_M 0x7
> +
> +/* PKI_BELTYPE_E_t */
> +enum cvmx_pki_beltype {
> +	CVMX_PKI_BELTYPE_NONE = PKI_BELTYPE_E__NONE_M,
> +	CVMX_PKI_BELTYPE_MISC = PKI_BELTYPE_E__MISC_M,
> +	CVMX_PKI_BELTYPE_IP4 = PKI_BELTYPE_E__IP4_M,
> +	CVMX_PKI_BELTYPE_IP6 = PKI_BELTYPE_E__IP6_M,
> +	CVMX_PKI_BELTYPE_TCP = PKI_BELTYPE_E__TCP_M,
> +	CVMX_PKI_BELTYPE_UDP = PKI_BELTYPE_E__UDP_M,
> +	CVMX_PKI_BELTYPE_SCTP = PKI_BELTYPE_E__SCTP_M,
> +	CVMX_PKI_BELTYPE_SNAP = PKI_BELTYPE_E__SNAP_M,
> +	CVMX_PKI_BELTYPE_MAX = CVMX_PKI_BELTYPE_SNAP
> +};
> +
> +struct cvmx_pki_frame_len {
> +	u16 maxlen;
> +	u16 minlen;
> +};
> +
> +struct cvmx_pki_tag_fields {
> +	u64 layer_g_src : 1;
> +	u64 layer_f_src : 1;
> +	u64 layer_e_src : 1;
> +	u64 layer_d_src : 1;
> +	u64 layer_c_src : 1;
> +	u64 layer_b_src : 1;
> +	u64 layer_g_dst : 1;
> +	u64 layer_f_dst : 1;
> +	u64 layer_e_dst : 1;
> +	u64 layer_d_dst : 1;
> +	u64 layer_c_dst : 1;
> +	u64 layer_b_dst : 1;
> +	u64 input_port : 1;
> +	u64 mpls_label : 1;
> +	u64 first_vlan : 1;
> +	u64 second_vlan : 1;
> +	u64 ip_prot_nexthdr : 1;
> +	u64 tag_sync : 1;
> +	u64 tag_spi : 1;
> +	u64 tag_gtp : 1;
> +	u64 tag_vni : 1;
> +};
> +
> +struct cvmx_pki_pkind_parse {
> +	u64 mpls_en : 1;
> +	u64 inst_hdr : 1;
> +	u64 lg_custom : 1;
> +	u64 fulc_en : 1;
> +	u64 dsa_en : 1;
> +	u64 hg2_en : 1;
> +	u64 hg_en : 1;
> +};
> +
> +struct cvmx_pki_pool_config {
> +	int pool_num;
> +	cvmx_fpa3_pool_t pool;
> +	u64 buffer_size;
> +	u64 buffer_count;
> +};
> +
> +struct cvmx_pki_qpg_config {
> +	int qpg_base;
> +	int port_add;
> +	int aura_num;
> +	int grp_ok;
> +	int grp_bad;
> +	int grptag_ok;
> +	int grptag_bad;
> +};
> +
> +struct cvmx_pki_aura_config {
> +	int aura_num;
> +	int pool_num;
> +	cvmx_fpa3_pool_t pool;
> +	cvmx_fpa3_gaura_t aura;
> +	int buffer_count;
> +};
> +
> +struct cvmx_pki_cluster_grp_config {
> +	int grp_num;
> +	u64 cluster_mask; /* Bit mask of cluster assigned to this
> cluster group */
> +};
> +
> +struct cvmx_pki_sso_grp_config {
> +	int group;
> +	int priority;
> +	int weight;
> +	int affinity;
> +	u64 core_mask;
> +	u8 core_mask_set;
> +};
> +
> +/* This is per style structure for configuring port parameters,
> + * it is kind of of profile which can be assigned to any port.
> + * If multiple ports are assigned same style be aware that modifying
> + * that style will modify the respective parameters for all the
> ports
> + * which are using this style
> + */
> +struct cvmx_pki_style_parm {
> +	bool ip6_udp_opt;
> +	bool lenerr_en;
> +	bool maxerr_en;
> +	bool minerr_en;
> +	u8 lenerr_eqpad;
> +	u8 minmax_sel;
> +	bool qpg_dis_grptag;
> +	bool fcs_strip;
> +	bool fcs_chk;
> +	bool rawdrp;
> +	bool force_drop;
> +	bool nodrop;
> +	bool qpg_dis_padd;
> +	bool qpg_dis_grp;
> +	bool qpg_dis_aura;
> +	u16 qpg_base;
> +	enum cvmx_pki_qpg_qos qpg_qos;
> +	u8 qpg_port_sh;
> +	u8 qpg_port_msb;
> +	u8 apad_nip;
> +	u8 wqe_vs;
> +	enum cvmx_sso_tag_type tag_type;
> +	bool pkt_lend;
> +	u8 wqe_hsz;
> +	u16 wqe_skip;
> +	u16 first_skip;
> +	u16 later_skip;
> +	enum cvmx_pki_cache_mode cache_mode;
> +	u8 dis_wq_dat;
> +	u64 mbuff_size;
> +	bool len_lg;
> +	bool len_lf;
> +	bool len_le;
> +	bool len_ld;
> +	bool len_lc;
> +	bool len_lb;
> +	bool csum_lg;
> +	bool csum_lf;
> +	bool csum_le;
> +	bool csum_ld;
> +	bool csum_lc;
> +	bool csum_lb;
> +};
> +
> +/* This is per style structure for configuring port's tag
> configuration,
> + * it is kind of of profile which can be assigned to any port.
> + * If multiple ports are assigned same style be aware that modiying
> that style
> + * will modify the respective parameters for all the ports which are
> + * using this style */
> +enum cvmx_pki_mtag_ptrsel {
> +	CVMX_PKI_MTAG_PTRSEL_SOP = 0,
> +	CVMX_PKI_MTAG_PTRSEL_LA = 8,
> +	CVMX_PKI_MTAG_PTRSEL_LB = 9,
> +	CVMX_PKI_MTAG_PTRSEL_LC = 10,
> +	CVMX_PKI_MTAG_PTRSEL_LD = 11,
> +	CVMX_PKI_MTAG_PTRSEL_LE = 12,
> +	CVMX_PKI_MTAG_PTRSEL_LF = 13,
> +	CVMX_PKI_MTAG_PTRSEL_LG = 14,
> +	CVMX_PKI_MTAG_PTRSEL_VL = 15,
> +};
> +
> +struct cvmx_pki_mask_tag {
> +	bool enable;
> +	int base;   /* CVMX_PKI_MTAG_PTRSEL_XXX */
> +	int offset; /* Offset from base. */
> +	u64 val;    /* Bitmask:
> +		1 = enable, 0 = disabled for each byte in the 64-byte
> array.*/
> +};
> +
> +struct cvmx_pki_style_tag_cfg {
> +	struct cvmx_pki_tag_fields tag_fields;
> +	struct cvmx_pki_mask_tag mask_tag[4];
> +};
> +
> +struct cvmx_pki_style_config {
> +	struct cvmx_pki_style_parm parm_cfg;
> +	struct cvmx_pki_style_tag_cfg tag_cfg;
> +};
> +
> +struct cvmx_pki_pkind_config {
> +	u8 cluster_grp;
> +	bool fcs_pres;
> +	struct cvmx_pki_pkind_parse parse_en;
> +	enum cvmx_pki_pkind_parse_mode initial_parse_mode;
> +	u8 fcs_skip;
> +	u8 inst_skip;
> +	int initial_style;
> +	bool custom_l2_hdr;
> +	u8 l2_scan_offset;
> +	u64 lg_scan_offset;
> +};
> +
> +struct cvmx_pki_port_config {
> +	struct cvmx_pki_pkind_config pkind_cfg;
> +	struct cvmx_pki_style_config style_cfg;
> +};
> +
> +struct cvmx_pki_global_parse {
> +	u64 virt_pen : 1;
> +	u64 clg_pen : 1;
> +	u64 cl2_pen : 1;
> +	u64 l4_pen : 1;
> +	u64 il3_pen : 1;
> +	u64 l3_pen : 1;
> +	u64 mpls_pen : 1;
> +	u64 fulc_pen : 1;
> +	u64 dsa_pen : 1;
> +	u64 hg_pen : 1;
> +};
> +
> +struct cvmx_pki_tag_sec {
> +	u16 dst6;
> +	u16 src6;
> +	u16 dst;
> +	u16 src;
> +};
> +
> +struct cvmx_pki_global_config {
> +	u64 cluster_mask[CVMX_PKI_NUM_CLUSTER_GROUP_MAX];
> +	enum cvmx_pki_stats_mode stat_mode;
> +	enum cvmx_pki_fpa_wait fpa_wait;
> +	struct cvmx_pki_global_parse gbl_pen;
> +	struct cvmx_pki_tag_sec tag_secret;
> +	struct cvmx_pki_frame_len frm_len[CVMX_PKI_NUM_FRAME_CHECK];
> +	enum cvmx_pki_beltype ltype_map[CVMX_PKI_NUM_BELTYPE];
> +	int pki_enable;
> +};
> +
> +#define CVMX_PKI_PCAM_TERM_E_NONE_M	 0x0
> +#define CVMX_PKI_PCAM_TERM_E_L2_CUSTOM_M 0x2
> +#define CVMX_PKI_PCAM_TERM_E_HIGIGD_M	 0x4
> +#define CVMX_PKI_PCAM_TERM_E_HIGIG_M	 0x5
> +#define CVMX_PKI_PCAM_TERM_E_SMACH_M	 0x8
> +#define CVMX_PKI_PCAM_TERM_E_SMACL_M	 0x9
> +#define CVMX_PKI_PCAM_TERM_E_DMACH_M	 0xA
> +#define CVMX_PKI_PCAM_TERM_E_DMACL_M	 0xB
> +#define CVMX_PKI_PCAM_TERM_E_GLORT_M	 0x12
> +#define CVMX_PKI_PCAM_TERM_E_DSA_M	 0x13
> +#define CVMX_PKI_PCAM_TERM_E_ETHTYPE0_M	 0x18
> +#define CVMX_PKI_PCAM_TERM_E_ETHTYPE1_M	 0x19
> +#define CVMX_PKI_PCAM_TERM_E_ETHTYPE2_M	 0x1A
> +#define CVMX_PKI_PCAM_TERM_E_ETHTYPE3_M	 0x1B
> +#define CVMX_PKI_PCAM_TERM_E_MPLS0_M	 0x1E
> +#define CVMX_PKI_PCAM_TERM_E_L3_SIPHH_M	 0x1F
> +#define CVMX_PKI_PCAM_TERM_E_L3_SIPMH_M	 0x20
> +#define CVMX_PKI_PCAM_TERM_E_L3_SIPML_M	 0x21
> +#define CVMX_PKI_PCAM_TERM_E_L3_SIPLL_M	 0x22
> +#define CVMX_PKI_PCAM_TERM_E_L3_FLAGS_M	 0x23
> +#define CVMX_PKI_PCAM_TERM_E_L3_DIPHH_M	 0x24
> +#define CVMX_PKI_PCAM_TERM_E_L3_DIPMH_M	 0x25
> +#define CVMX_PKI_PCAM_TERM_E_L3_DIPML_M	 0x26
> +#define CVMX_PKI_PCAM_TERM_E_L3_DIPLL_M	 0x27
> +#define CVMX_PKI_PCAM_TERM_E_LD_VNI_M	 0x28
> +#define CVMX_PKI_PCAM_TERM_E_IL3_FLAGS_M 0x2B
> +#define CVMX_PKI_PCAM_TERM_E_LF_SPI_M	 0x2E
> +#define CVMX_PKI_PCAM_TERM_E_L4_SPORT_M	 0x2f
> +#define CVMX_PKI_PCAM_TERM_E_L4_PORT_M	 0x30
> +#define CVMX_PKI_PCAM_TERM_E_LG_CUSTOM_M 0x39
> +
> +enum cvmx_pki_term {
> +	CVMX_PKI_PCAM_TERM_NONE = CVMX_PKI_PCAM_TERM_E_NONE_M,
> +	CVMX_PKI_PCAM_TERM_L2_CUSTOM =
> CVMX_PKI_PCAM_TERM_E_L2_CUSTOM_M,
> +	CVMX_PKI_PCAM_TERM_HIGIGD = CVMX_PKI_PCAM_TERM_E_HIGIGD_M,
> +	CVMX_PKI_PCAM_TERM_HIGIG = CVMX_PKI_PCAM_TERM_E_HIGIG_M,
> +	CVMX_PKI_PCAM_TERM_SMACH = CVMX_PKI_PCAM_TERM_E_SMACH_M,
> +	CVMX_PKI_PCAM_TERM_SMACL = CVMX_PKI_PCAM_TERM_E_SMACL_M,
> +	CVMX_PKI_PCAM_TERM_DMACH = CVMX_PKI_PCAM_TERM_E_DMACH_M,
> +	CVMX_PKI_PCAM_TERM_DMACL = CVMX_PKI_PCAM_TERM_E_DMACL_M,
> +	CVMX_PKI_PCAM_TERM_GLORT = CVMX_PKI_PCAM_TERM_E_GLORT_M,
> +	CVMX_PKI_PCAM_TERM_DSA = CVMX_PKI_PCAM_TERM_E_DSA_M,
> +	CVMX_PKI_PCAM_TERM_ETHTYPE0 = CVMX_PKI_PCAM_TERM_E_ETHTYPE0_M,
> +	CVMX_PKI_PCAM_TERM_ETHTYPE1 = CVMX_PKI_PCAM_TERM_E_ETHTYPE1_M,
> +	CVMX_PKI_PCAM_TERM_ETHTYPE2 = CVMX_PKI_PCAM_TERM_E_ETHTYPE2_M,
> +	CVMX_PKI_PCAM_TERM_ETHTYPE3 = CVMX_PKI_PCAM_TERM_E_ETHTYPE3_M,
> +	CVMX_PKI_PCAM_TERM_MPLS0 = CVMX_PKI_PCAM_TERM_E_MPLS0_M,
> +	CVMX_PKI_PCAM_TERM_L3_SIPHH = CVMX_PKI_PCAM_TERM_E_L3_SIPHH_M,
> +	CVMX_PKI_PCAM_TERM_L3_SIPMH = CVMX_PKI_PCAM_TERM_E_L3_SIPMH_M,
> +	CVMX_PKI_PCAM_TERM_L3_SIPML = CVMX_PKI_PCAM_TERM_E_L3_SIPML_M,
> +	CVMX_PKI_PCAM_TERM_L3_SIPLL = CVMX_PKI_PCAM_TERM_E_L3_SIPLL_M,
> +	CVMX_PKI_PCAM_TERM_L3_FLAGS = CVMX_PKI_PCAM_TERM_E_L3_FLAGS_M,
> +	CVMX_PKI_PCAM_TERM_L3_DIPHH = CVMX_PKI_PCAM_TERM_E_L3_DIPHH_M,
> +	CVMX_PKI_PCAM_TERM_L3_DIPMH = CVMX_PKI_PCAM_TERM_E_L3_DIPMH_M,
> +	CVMX_PKI_PCAM_TERM_L3_DIPML = CVMX_PKI_PCAM_TERM_E_L3_DIPML_M,
> +	CVMX_PKI_PCAM_TERM_L3_DIPLL = CVMX_PKI_PCAM_TERM_E_L3_DIPLL_M,
> +	CVMX_PKI_PCAM_TERM_LD_VNI = CVMX_PKI_PCAM_TERM_E_LD_VNI_M,
> +	CVMX_PKI_PCAM_TERM_IL3_FLAGS =
> CVMX_PKI_PCAM_TERM_E_IL3_FLAGS_M,
> +	CVMX_PKI_PCAM_TERM_LF_SPI = CVMX_PKI_PCAM_TERM_E_LF_SPI_M,
> +	CVMX_PKI_PCAM_TERM_L4_PORT = CVMX_PKI_PCAM_TERM_E_L4_PORT_M,
> +	CVMX_PKI_PCAM_TERM_L4_SPORT = CVMX_PKI_PCAM_TERM_E_L4_SPORT_M,
> +	CVMX_PKI_PCAM_TERM_LG_CUSTOM = CVMX_PKI_PCAM_TERM_E_LG_CUSTOM_M
> +};
> +
> +#define CVMX_PKI_DMACH_SHIFT	  32
> +#define CVMX_PKI_DMACH_MASK	  cvmx_build_mask(16)
> +#define CVMX_PKI_DMACL_MASK	  CVMX_PKI_DATA_MASK_32
> +#define CVMX_PKI_DATA_MASK_32	  cvmx_build_mask(32)
> +#define CVMX_PKI_DATA_MASK_16	  cvmx_build_mask(16)
> +#define CVMX_PKI_DMAC_MATCH_EXACT cvmx_build_mask(48)
> +
> +struct cvmx_pki_pcam_input {
> +	u64 style;
> +	u64 style_mask; /* bits: 1-match, 0-dont care */
> +	enum cvmx_pki_term field;
> +	u32 field_mask; /* bits: 1-match, 0-dont care */
> +	u64 data;
> +	u64 data_mask; /* bits: 1-match, 0-dont care */
> +};
> +
> +struct cvmx_pki_pcam_action {
> +	enum cvmx_pki_parse_mode_chg parse_mode_chg;
> +	enum cvmx_pki_layer_type layer_type_set;
> +	int style_add;
> +	int parse_flag_set;
> +	int pointer_advance;
> +};
> +
> +struct cvmx_pki_pcam_config {
> +	int in_use;
> +	int entry_num;
> +	u64 cluster_mask;
> +	struct cvmx_pki_pcam_input pcam_input;
> +	struct cvmx_pki_pcam_action pcam_action;
> +};
> +
> +/**
> + * Status statistics for a port
> + */
> +struct cvmx_pki_port_stats {
> +	u64 dropped_octets;
> +	u64 dropped_packets;
> +	u64 pci_raw_packets;
> +	u64 octets;
> +	u64 packets;
> +	u64 multicast_packets;
> +	u64 broadcast_packets;
> +	u64 len_64_packets;
> +	u64 len_65_127_packets;
> +	u64 len_128_255_packets;
> +	u64 len_256_511_packets;
> +	u64 len_512_1023_packets;
> +	u64 len_1024_1518_packets;
> +	u64 len_1519_max_packets;
> +	u64 fcs_align_err_packets;
> +	u64 runt_packets;
> +	u64 runt_crc_packets;
> +	u64 oversize_packets;
> +	u64 oversize_crc_packets;
> +	u64 inb_packets;
> +	u64 inb_octets;
> +	u64 inb_errors;
> +	u64 mcast_l2_red_packets;
> +	u64 bcast_l2_red_packets;
> +	u64 mcast_l3_red_packets;
> +	u64 bcast_l3_red_packets;
> +};
> +
> +/**
> + * PKI Packet Instruction Header Structure (PKI_INST_HDR_S)
> + */
> +typedef union {
> +	u64 u64;
> +	struct {
> +		u64 w : 1;    /* INST_HDR size: 0 = 2 bytes, 1 = 4 or 8
> bytes */
> +		u64 raw : 1;  /* RAW packet indicator in WQE[RAW]: 1 =
> enable */
> +		u64 utag : 1; /* Use INST_HDR[TAG] to compute WQE[TAG]:
> 1 = enable */
> +		u64 uqpg : 1; /* Use INST_HDR[QPG] to compute QPG: 1 =
> enable */
> +		u64 rsvd1 : 1;
> +		u64 pm : 3; /* Packet parsing mode. Legal values =
> 0x0..0x7 */
> +		u64 sl : 8; /* Number of bytes in INST_HDR. */
> +		/* The following fields are not present, if INST_HDR[W]
> = 0: */
> +		u64 utt : 1; /* Use INST_HDR[TT] to compute WQE[TT]: 1
> = enable */
> +		u64 tt : 2;  /* INST_HDR[TT] => WQE[TT], if
> INST_HDR[UTT] = 1 */
> +		u64 rsvd2 : 2;
> +		u64 qpg : 11; /* INST_HDR[QPG] => QPG, if
> INST_HDR[UQPG] = 1 */
> +		u64 tag : 32; /* INST_HDR[TAG] => WQE[TAG], if
> INST_HDR[UTAG] = 1 */
> +	} s;
> +} cvmx_pki_inst_hdr_t;
> +
> +/**
> + * This function assignes the clusters to a group, later pkind can
> be
> + * configured to use that group depending on number of clusters
> pkind
> + * would use. A given cluster can only be enabled in a single
> cluster group.
> + * Number of clusters assign to that group determines how many
> engine can work
> + * in parallel to process the packet. Eack cluster can process x
> MPPS.
> + *
> + * @param node	Node
> + * @param cluster_group Group to attach clusters to.
> + * @param cluster_mask The mask of clusters which needs to be
> assigned to the group.
> + */
> +static inline int cvmx_pki_attach_cluster_to_group(int node, u64
> cluster_group, u64 cluster_mask)
> +{
> +	cvmx_pki_icgx_cfg_t pki_cl_grp;
> +
> +	if (cluster_group >= CVMX_PKI_NUM_CLUSTER_GROUP) {
> +		debug("ERROR: config cluster group %d",
> (int)cluster_group);
> +		return -1;
> +	}
> +	pki_cl_grp.u64 = cvmx_read_csr_node(node,
> CVMX_PKI_ICGX_CFG(cluster_group));
> +	pki_cl_grp.s.clusters = cluster_mask;
> +	cvmx_write_csr_node(node, CVMX_PKI_ICGX_CFG(cluster_group),
> pki_cl_grp.u64);
> +	return 0;
> +}
> +
> +static inline void cvmx_pki_write_global_parse(int node, struct
> cvmx_pki_global_parse gbl_pen)
> +{
> +	cvmx_pki_gbl_pen_t gbl_pen_reg;
> +
> +	gbl_pen_reg.u64 = cvmx_read_csr_node(node, CVMX_PKI_GBL_PEN);
> +	gbl_pen_reg.s.virt_pen = gbl_pen.virt_pen;
> +	gbl_pen_reg.s.clg_pen = gbl_pen.clg_pen;
> +	gbl_pen_reg.s.cl2_pen = gbl_pen.cl2_pen;
> +	gbl_pen_reg.s.l4_pen = gbl_pen.l4_pen;
> +	gbl_pen_reg.s.il3_pen = gbl_pen.il3_pen;
> +	gbl_pen_reg.s.l3_pen = gbl_pen.l3_pen;
> +	gbl_pen_reg.s.mpls_pen = gbl_pen.mpls_pen;
> +	gbl_pen_reg.s.fulc_pen = gbl_pen.fulc_pen;
> +	gbl_pen_reg.s.dsa_pen = gbl_pen.dsa_pen;
> +	gbl_pen_reg.s.hg_pen = gbl_pen.hg_pen;
> +	cvmx_write_csr_node(node, CVMX_PKI_GBL_PEN, gbl_pen_reg.u64);
> +}
> +
> +static inline void cvmx_pki_write_tag_secret(int node, struct
> cvmx_pki_tag_sec tag_secret)
> +{
> +	cvmx_pki_tag_secret_t tag_secret_reg;
> +
> +	tag_secret_reg.u64 = cvmx_read_csr_node(node,
> CVMX_PKI_TAG_SECRET);
> +	tag_secret_reg.s.dst6 = tag_secret.dst6;
> +	tag_secret_reg.s.src6 = tag_secret.src6;
> +	tag_secret_reg.s.dst = tag_secret.dst;
> +	tag_secret_reg.s.src = tag_secret.src;
> +	cvmx_write_csr_node(node, CVMX_PKI_TAG_SECRET,
> tag_secret_reg.u64);
> +}
> +
> +static inline void cvmx_pki_write_ltype_map(int node, enum
> cvmx_pki_layer_type layer,
> +					    enum cvmx_pki_beltype
> backend)
> +{
> +	cvmx_pki_ltypex_map_t ltype_map;
> +
> +	if (layer > CVMX_PKI_LTYPE_E_MAX || backend >
> CVMX_PKI_BELTYPE_MAX) {
> +		debug("ERROR: invalid ltype beltype mapping\n");
> +		return;
> +	}
> +	ltype_map.u64 = cvmx_read_csr_node(node,
> CVMX_PKI_LTYPEX_MAP(layer));
> +	ltype_map.s.beltype = backend;
> +	cvmx_write_csr_node(node, CVMX_PKI_LTYPEX_MAP(layer),
> ltype_map.u64);
> +}
> +
> +/**
> + * This function enables the cluster group to start parsing.
> + *
> + * @param node    Node number.
> + * @param cl_grp  Cluster group to enable parsing.
> + */
> +static inline int cvmx_pki_parse_enable(int node, unsigned int
> cl_grp)
> +{
> +	cvmx_pki_icgx_cfg_t pki_cl_grp;
> +
> +	if (cl_grp >= CVMX_PKI_NUM_CLUSTER_GROUP) {
> +		debug("ERROR: pki parse en group %d", (int)cl_grp);
> +		return -1;
> +	}
> +	pki_cl_grp.u64 = cvmx_read_csr_node(node,
> CVMX_PKI_ICGX_CFG(cl_grp));
> +	pki_cl_grp.s.pena = 1;
> +	cvmx_write_csr_node(node, CVMX_PKI_ICGX_CFG(cl_grp),
> pki_cl_grp.u64);
> +	return 0;
> +}
> +
> +/**
> + * This function enables the PKI to send bpid level backpressure to
> CN78XX inputs.
> + *
> + * @param node Node number.
> + */
> +static inline void cvmx_pki_enable_backpressure(int node)
> +{
> +	cvmx_pki_buf_ctl_t pki_buf_ctl;
> +
> +	pki_buf_ctl.u64 = cvmx_read_csr_node(node, CVMX_PKI_BUF_CTL);
> +	pki_buf_ctl.s.pbp_en = 1;
> +	cvmx_write_csr_node(node, CVMX_PKI_BUF_CTL, pki_buf_ctl.u64);
> +}
> +
> +/**
> + * Clear the statistics counters for a port.
> + *
> + * @param node Node number.
> + * @param port Port number (ipd_port) to get statistics for.
> + *    Make sure PKI_STATS_CTL:mode is set to 0 for collecting per
> port/pkind stats.
> + */
> +void cvmx_pki_clear_port_stats(int node, u64 port);
> +
> +/**
> + * Get the status counters for index from PKI.
> + *
> + * @param node	  Node number.
> + * @param index   PKIND number, if PKI_STATS_CTL:mode = 0 or
> + *     style(flow) number, if PKI_STATS_CTL:mode = 1
> + * @param status  Where to put the results.
> + */
> +void cvmx_pki_get_stats(int node, int index, struct
> cvmx_pki_port_stats *status);
> +
> +/**
> + * Get the statistics counters for a port.
> + *
> + * @param node	 Node number
> + * @param port   Port number (ipd_port) to get statistics for.
> + *    Make sure PKI_STATS_CTL:mode is set to 0 for collecting per
> port/pkind stats.
> + * @param status Where to put the results.
> + */
> +static inline void cvmx_pki_get_port_stats(int node, u64 port,
> struct cvmx_pki_port_stats *status)
> +{
> +	int xipd = cvmx_helper_node_to_ipd_port(node, port);
> +	int xiface = cvmx_helper_get_interface_num(xipd);
> +	int index = cvmx_helper_get_interface_index_num(port);
> +	int pknd = cvmx_helper_get_pknd(xiface, index);
> +
> +	cvmx_pki_get_stats(node, pknd, status);
> +}
> +
> +/**
> + * Get the statistics counters for a flow represented by style in
> PKI.
> + *
> + * @param node Node number.
> + * @param style_num Style number to get statistics for.
> + *    Make sure PKI_STATS_CTL:mode is set to 1 for collecting per
> style/flow stats.
> + * @param status Where to put the results.
> + */
> +static inline void cvmx_pki_get_flow_stats(int node, u64 style_num,
> +					   struct cvmx_pki_port_stats
> *status)
> +{
> +	cvmx_pki_get_stats(node, style_num, status);
> +}
> +
> +/**
> + * Show integrated PKI configuration.
> + *
> + * @param node	   node number
> + */
> +int cvmx_pki_config_dump(unsigned int node);
> +
> +/**
> + * Show integrated PKI statistics.
> + *
> + * @param node	   node number
> + */
> +int cvmx_pki_stats_dump(unsigned int node);
> +
> +/**
> + * Clear PKI statistics.
> + *
> + * @param node	   node number
> + */
> +void cvmx_pki_stats_clear(unsigned int node);
> +
> +/**
> + * This function enables PKI.
> + *
> + * @param node	 node to enable pki in.
> + */
> +void cvmx_pki_enable(int node);
> +
> +/**
> + * This function disables PKI.
> + *
> + * @param node	node to disable pki in.
> + */
> +void cvmx_pki_disable(int node);
> +
> +/**
> + * This function soft resets PKI.
> + *
> + * @param node	node to enable pki in.
> + */
> +void cvmx_pki_reset(int node);
> +
> +/**
> + * This function sets the clusters in PKI.
> + *
> + * @param node	node to set clusters in.
> + */
> +int cvmx_pki_setup_clusters(int node);
> +
> +/**
> + * This function reads global configuration of PKI block.
> + *
> + * @param node    Node number.
> + * @param gbl_cfg Pointer to struct to read global configuration
> + */
> +void cvmx_pki_read_global_config(int node, struct
> cvmx_pki_global_config *gbl_cfg);
> +
> +/**
> + * This function writes global configuration of PKI into hw.
> + *
> + * @param node    Node number.
> + * @param gbl_cfg Pointer to struct to global configuration
> + */
> +void cvmx_pki_write_global_config(int node, struct
> cvmx_pki_global_config *gbl_cfg);
> +
> +/**
> + * This function reads per pkind parameters in hardware which
> defines how
> + * the incoming packet is processed.
> + *
> + * @param node   Node number.
> + * @param pkind  PKI supports a large number of incoming interfaces
> and packets
> + *     arriving on different interfaces or channels may want to be
> processed
> + *     differently. PKI uses the pkind to determine how the incoming
> packet
> + *     is processed.
> + * @param pkind_cfg	Pointer to struct conatining pkind
> configuration read
> + *     from hardware.
> + */
> +int cvmx_pki_read_pkind_config(int node, int pkind, struct
> cvmx_pki_pkind_config *pkind_cfg);
> +
> +/**
> + * This function writes per pkind parameters in hardware which
> defines how
> + * the incoming packet is processed.
> + *
> + * @param node   Node number.
> + * @param pkind  PKI supports a large number of incoming interfaces
> and packets
> + *     arriving on different interfaces or channels may want to be
> processed
> + *     differently. PKI uses the pkind to determine how the incoming
> packet
> + *     is processed.
> + * @param pkind_cfg	Pointer to struct conatining pkind
> configuration need
> + *     to be written in hardware.
> + */
> +int cvmx_pki_write_pkind_config(int node, int pkind, struct
> cvmx_pki_pkind_config *pkind_cfg);
> +
> +/**
> + * This function reads parameters associated with tag configuration
> in hardware.
> + *
> + * @param node	 Node number.
> + * @param style  Style to configure tag for.
> + * @param cluster_mask  Mask of clusters to configure the style for.
> + * @param tag_cfg  Pointer to tag configuration struct.
> + */
> +void cvmx_pki_read_tag_config(int node, int style, u64 cluster_mask,
> +			      struct cvmx_pki_style_tag_cfg *tag_cfg);
> +
> +/**
> + * This function writes/configures parameters associated with tag
> + * configuration in hardware.
> + *
> + * @param node  Node number.
> + * @param style  Style to configure tag for.
> + * @param cluster_mask  Mask of clusters to configure the style for.
> + * @param tag_cfg  Pointer to taf configuration struct.
> + */
> +void cvmx_pki_write_tag_config(int node, int style, u64
> cluster_mask,
> +			       struct cvmx_pki_style_tag_cfg *tag_cfg);
> +
> +/**
> + * This function reads parameters associated with style in hardware.
> + *
> + * @param node	Node number.
> + * @param style  Style to read from.
> + * @param cluster_mask  Mask of clusters style belongs to.
> + * @param style_cfg  Pointer to style config struct.
> + */
> +void cvmx_pki_read_style_config(int node, int style, u64
> cluster_mask,
> +				struct cvmx_pki_style_config
> *style_cfg);
> +
> +/**
> + * This function writes/configures parameters associated with style
> in hardware.
> + *
> + * @param node  Node number.
> + * @param style  Style to configure.
> + * @param cluster_mask  Mask of clusters to configure the style for.
> + * @param style_cfg  Pointer to style config struct.
> + */
> +void cvmx_pki_write_style_config(int node, u64 style, u64
> cluster_mask,
> +				 struct cvmx_pki_style_config
> *style_cfg);
> +/**
> + * This function reads qpg entry at specified offset from qpg table
> + *
> + * @param node  Node number.
> + * @param offset  Offset in qpg table to read from.
> + * @param qpg_cfg  Pointer to structure containing qpg values
> + */
> +int cvmx_pki_read_qpg_entry(int node, int offset, struct
> cvmx_pki_qpg_config *qpg_cfg);
> +
> +/**
> + * This function writes qpg entry at specified offset in qpg table
> + *
> + * @param node  Node number.
> + * @param offset  Offset in qpg table to write to.
> + * @param qpg_cfg  Pointer to stricture containing qpg values.
> + */
> +void cvmx_pki_write_qpg_entry(int node, int offset, struct
> cvmx_pki_qpg_config *qpg_cfg);
> +
> +/**
> + * This function writes pcam entry at given offset in pcam table in
> hardware
> + *
> + * @param node  Node number.
> + * @param index	 Offset in pcam table.
> + * @param cluster_mask  Mask of clusters in which to write pcam
> entry.
> + * @param input  Input keys to pcam match passed as struct.
> + * @param action  PCAM match action passed as struct
> + */
> +int cvmx_pki_pcam_write_entry(int node, int index, u64 cluster_mask,
> +			      struct cvmx_pki_pcam_input input, struct
> cvmx_pki_pcam_action action);
> +/**
> + * Configures the channel which will receive backpressure from the
> specified bpid.
> + * Each channel listens for backpressure on a specific bpid.
> + * Each bpid can backpressure multiple channels.
> + * @param node  Node number.
> + * @param bpid  BPID from which channel will receive backpressure.
> + * @param channel  Channel number to receive backpressue.
> + */
> +int cvmx_pki_write_channel_bpid(int node, int channel, int bpid);
> +
> +/**
> + * Configures the bpid on which, specified channel will
> + * assert backpressure.
> + * Each bpid receives backpressure from auras.
> + * Multiple auras can backpressure single bpid.
> + * @param node  Node number.
> + * @param aura  Number which will assert backpressure on that bpid.
> + * @param bpid  To assert backpressure on.
> + */
> +int cvmx_pki_write_aura_bpid(int node, int aura, int bpid);
> +
> +/**
> + * Enables/Disabled QoS (RED Drop, Tail Drop & backpressure) for
> the* PKI aura.
> + *
> + * @param node  Node number
> + * @param aura  To enable/disable QoS on.
> + * @param ena_red  Enable/Disable RED drop between pass and drop
> level
> + *    1-enable 0-disable
> + * @param ena_drop  Enable/disable tail drop when max drop level
> exceeds
> + *    1-enable 0-disable
> + * @param ena_bp  Enable/Disable asserting backpressure on bpid when
> + *    max DROP level exceeds.
> + *    1-enable 0-disable
> + */
> +int cvmx_pki_enable_aura_qos(int node, int aura, bool ena_red, bool
> ena_drop, bool ena_bp);
> +
> +/**
> + * This function gives the initial style used by that pkind.
> + *
> + * @param node  Node number.
> + * @param pkind  PKIND number.
> + */
> +int cvmx_pki_get_pkind_style(int node, int pkind);
> +
> +/**
> + * This function sets the wqe buffer mode. First packet data buffer
> can reside
> + * either in same buffer as wqe OR it can go in separate buffer. If
> used the later mode,
> + * make sure software allocate enough buffers to now have wqe
> separate from packet data.
> + *
> + * @param node  Node number.
> + * @param style  Style to configure.
> + * @param pkt_outside_wqe
> + *    0 = The packet link pointer will be at word [FIRST_SKIP]
> immediately
> + *    followed by packet data, in the same buffer as the work queue
> entry.
> + *    1 = The packet link pointer will be at word [FIRST_SKIP] in a
> new
> + *    buffer separate from the work queue entry. Words following the
> + *    WQE in the same cache line will be zeroed, other lines in the
> + *    buffer will not be modified and will retain stale data (from
> the
> + *    buffer’s previous use). This setting may decrease the peak PKI
> + *    performance by up to half on small packets.
> + */
> +void cvmx_pki_set_wqe_mode(int node, u64 style, bool
> pkt_outside_wqe);
> +
> +/**
> + * This function sets the Packet mode of all ports and styles to
> little-endian.
> + * It Changes write operations of packet data to L2C to
> + * be in little-endian. Does not change the WQE header format, which
> is
> + * properly endian neutral.
> + *
> + * @param node  Node number.
> + * @param style  Style to configure.
> + */
> +void cvmx_pki_set_little_endian(int node, u64 style);
> +
> +/**
> + * Enables/Disables L2 length error check and max & min frame length
> checks.
> + *
> + * @param node  Node number.
> + * @param pknd  PKIND to disable error for.
> + * @param l2len_err	 L2 length error check enable.
> + * @param maxframe_err	Max frame error check enable.
> + * @param minframe_err	Min frame error check enable.
> + *    1 -- Enabel err checks
> + *    0 -- Disable error checks
> + */
> +void cvmx_pki_endis_l2_errs(int node, int pknd, bool l2len_err, bool
> maxframe_err,
> +			    bool minframe_err);
> +
> +/**
> + * Enables/Disables fcs check and fcs stripping on the pkind.
> + *
> + * @param node  Node number.
> + * @param pknd  PKIND to apply settings on.
> + * @param fcs_chk  Enable/disable fcs check.
> + *    1 -- enable fcs error check.
> + *    0 -- disable fcs error check.
> + * @param fcs_strip	 Strip L2 FCS bytes from packet, decrease
> WQE[LEN] by 4 bytes
> + *    1 -- strip L2 FCS.
> + *    0 -- Do not strip L2 FCS.
> + */
> +void cvmx_pki_endis_fcs_check(int node, int pknd, bool fcs_chk, bool
> fcs_strip);
> +
> +/**
> + * This function shows the qpg table entries, read directly from
> hardware.
> + *
> + * @param node  Node number.
> + * @param num_entry  Number of entries to print.
> + */
> +void cvmx_pki_show_qpg_entries(int node, u16 num_entry);
> +
> +/**
> + * This function shows the pcam table in raw format read directly
> from hardware.
> + *
> + * @param node  Node number.
> + */
> +void cvmx_pki_show_pcam_entries(int node);
> +
> +/**
> + * This function shows the valid entries in readable format,
> + * read directly from hardware.
> + *
> + * @param node  Node number.
> + */
> +void cvmx_pki_show_valid_pcam_entries(int node);
> +
> +/**
> + * This function shows the pkind attributes in readable format,
> + * read directly from hardware.
> + * @param node  Node number.
> + * @param pkind  PKIND number to print.
> + */
> +void cvmx_pki_show_pkind_attributes(int node, int pkind);
> +
> +/**
> + * @INTERNAL
> + * This function is called by cvmx_helper_shutdown() to extract all
> FPA buffers
> + * out of the PKI. After this function completes, all FPA buffers
> that were
> + * prefetched by PKI will be in the appropriate FPA pool.
> + * This functions does not reset the PKI.
> + * WARNING: It is very important that PKI be reset soon after a call
> to this function.
> + *
> + * @param node  Node number.
> + */
> +void __cvmx_pki_free_ptr(int node);
> +
> +#endif
> diff --git a/arch/mips/mach-octeon/include/mach/cvmx-pko-internal-
> ports-range.h b/arch/mips/mach-octeon/include/mach/cvmx-pko-internal-
> ports-range.h
> new file mode 100644
> index 000000000000..1fb49b3fb6de
> --- /dev/null
> +++ b/arch/mips/mach-octeon/include/mach/cvmx-pko-internal-ports-
> range.h
> @@ -0,0 +1,43 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +/*
> + * Copyright (C) 2020 Marvell International Ltd.
> + */
> +
> +#ifndef __CVMX_INTERNAL_PORTS_RANGE__
> +#define __CVMX_INTERNAL_PORTS_RANGE__
> +
> +/*
> + * Allocated a block of internal ports for the specified
> interface/port
> + *
> + * @param  interface  the interface for which the internal ports are
> requested
> + * @param  port       the index of the port within in the interface
> for which the internal ports
> + *                    are requested.
> + * @param  count      the number of internal ports requested
> + *
> + * @return  0 on success
> + *         -1 on failure
> + */
> +int cvmx_pko_internal_ports_alloc(int interface, int port, u64
> count);
> +
> +/*
> + * Free the internal ports associated with the specified
> interface/port
> + *
> + * @param  interface  the interface for which the internal ports are
> requested
> + * @param  port       the index of the port within in the interface
> for which the internal ports
> + *                    are requested.
> + *
> + * @return  0 on success
> + *         -1 on failure
> + */
> +int cvmx_pko_internal_ports_free(int interface, int port);
> +
> +/*
> + * Frees up all the allocated internal ports.
> + */
> +void cvmx_pko_internal_ports_range_free_all(void);
> +
> +void cvmx_pko_internal_ports_range_show(void);
> +
> +int __cvmx_pko_internal_ports_range_init(void);
> +
> +#endif
> diff --git a/arch/mips/mach-octeon/include/mach/cvmx-pko3-queue.h
> b/arch/mips/mach-octeon/include/mach/cvmx-pko3-queue.h
> new file mode 100644
> index 000000000000..5f8398904953
> --- /dev/null
> +++ b/arch/mips/mach-octeon/include/mach/cvmx-pko3-queue.h
> @@ -0,0 +1,175 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +/*
> + * Copyright (C) 2020 Marvell International Ltd.
> + */
> +
> +#ifndef __CVMX_PKO3_QUEUE_H__
> +#define __CVMX_PKO3_QUEUE_H__
> +
> +/**
> + * @INTERNAL
> + *
> + * Find or allocate global port/dq map table
> + * which is a named table, contains entries for
> + * all possible OCI nodes.
> + *
> + * The table global pointer is stored in core-local variable
> + * so that every core will call this function once, on first use.
> + */
> +int __cvmx_pko3_dq_table_setup(void);
> +
> +/*
> + * Get the base Descriptor Queue number for an IPD port on the local
> node
> + */
> +int cvmx_pko3_get_queue_base(int ipd_port);
> +
> +/*
> + * Get the number of Descriptor Queues assigned for an IPD port
> + */
> +int cvmx_pko3_get_queue_num(int ipd_port);
> +
> +/**
> + * Get L1/Port Queue number assigned to interface port.
> + *
> + * @param xiface is interface number.
> + * @param index is port index.
> + */
> +int cvmx_pko3_get_port_queue(int xiface, int index);
> +
> +/*
> + * Configure L3 through L5 Scheduler Queues and Descriptor Queues
> + *
> + * The Scheduler Queues in Levels 3 to 5 and Descriptor Queues are
> + * configured one-to-one or many-to-one to a single parent Scheduler
> + * Queues. The level of the parent SQ is specified in an argument,
> + * as well as the number of children to attach to the specific
> parent.
> + * The children can have fair round-robin or priority-based
> scheduling
> + * when multiple children are assigned a single parent.
> + *
> + * @param node is the OCI node location for the queues to be
> configured
> + * @param parent_level is the level of the parent queue, 2 to 5.
> + * @param parent_queue is the number of the parent Scheduler Queue
> + * @param child_base is the number of the first child SQ or DQ to
> assign to
> + * @param parent
> + * @param child_count is the number of consecutive children to
> assign
> + * @param stat_prio_count is the priority setting for the children
> L2 SQs
> + *
> + * If <stat_prio_count> is -1, the Ln children will have equal
> Round-Robin
> + * relationship with eachother. If <stat_prio_count> is 0, all Ln
> children
> + * will be arranged in Weighted-Round-Robin, with the first having
> the most
> + * precedence. If <stat_prio_count> is between 1 and 8, it indicates
> how
> + * many children will have static priority settings (with the first
> having
> + * the most precedence), with the remaining Ln children having WRR
> scheduling.
> + *
> + * @returns 0 on success, -1 on failure.
> + *
> + * Note: this function supports the configuration of node-local
> unit.
> + */
> +int cvmx_pko3_sq_config_children(unsigned int node, unsigned int
> parent_level,
> +				 unsigned int parent_queue, unsigned
> int child_base,
> +				 unsigned int child_count, int
> stat_prio_count);
> +
> +/*
> + * @INTERNAL
> + * Register a range of Descriptor Queues wth an interface port
> + *
> + * This function poulates the DQ-to-IPD translation table
> + * used by the application to retrieve the DQ range (typically
> ordered
> + * by priority) for a given IPD-port, which is either a physical
> port,
> + * or a channel on a channelized interface (i.e. ILK).
> + *
> + * @param xiface is the physical interface number
> + * @param index is either a physical port on an interface
> + * @param or a channel of an ILK interface
> + * @param dq_base is the first Descriptor Queue number in a
> consecutive range
> + * @param dq_count is the number of consecutive Descriptor Queues
> leading
> + * @param the same channel or port.
> + *
> + * Only a consecurive range of Descriptor Queues can be associated
> with any
> + * given channel/port, and usually they are ordered from most to
> least
> + * in terms of scheduling priority.
> + *
> + * Note: thus function only populates the node-local translation
> table.
> + *
> + * @returns 0 on success, -1 on failure.
> + */
> +int __cvmx_pko3_ipd_dq_register(int xiface, int index, unsigned int
> dq_base, unsigned int dq_count);
> +
> +/**
> + * @INTERNAL
> + *
> + * Unregister DQs associated with CHAN_E (IPD port)
> + */
> +int __cvmx_pko3_ipd_dq_unregister(int xiface, int index);
> +
> +/*
> + * Map channel number in PKO
> + *
> + * @param node is to specify the node to which this configuration is
> applied.
> + * @param pq_num specifies the Port Queue (i.e. L1) queue number.
> + * @param l2_l3_q_num  specifies L2/L3 queue number.
> + * @param channel specifies the channel number to map to the queue.
> + *
> + * The channel assignment applies to L2 or L3 Shaper Queues
> depending
> + * on the setting of channel credit level.
> + *
> + * @return returns none.
> + */
> +void cvmx_pko3_map_channel(unsigned int node, unsigned int pq_num,
> unsigned int l2_l3_q_num,
> +			   u16 channel);
> +
> +int cvmx_pko3_pq_config(unsigned int node, unsigned int mac_num,
> unsigned int pq_num);
> +
> +int cvmx_pko3_port_cir_set(unsigned int node, unsigned int pq_num,
> unsigned long rate_kbips,
> +			   unsigned int burst_bytes, int adj_bytes);
> +int cvmx_pko3_dq_cir_set(unsigned int node, unsigned int pq_num,
> unsigned long rate_kbips,
> +			 unsigned int burst_bytes);
> +int cvmx_pko3_dq_pir_set(unsigned int node, unsigned int pq_num,
> unsigned long rate_kbips,
> +			 unsigned int burst_bytes);
> +typedef enum {
> +	CVMX_PKO3_SHAPE_RED_STALL,
> +	CVMX_PKO3_SHAPE_RED_DISCARD,
> +	CVMX_PKO3_SHAPE_RED_PASS
> +} red_action_t;
> +
> +void cvmx_pko3_dq_red(unsigned int node, unsigned int dq_num,
> red_action_t red_act,
> +		      int8_t len_adjust);
> +
> +/**
> + * Macros to deal with short floating point numbers,
> + * where unsigned exponent, and an unsigned normalized
> + * mantissa are represented each with a defined field width.
> + *
> + */
> +#define CVMX_SHOFT_MANT_BITS 8
> +#define CVMX_SHOFT_EXP_BITS  4
> +
> +/**
> + * Convert short-float to an unsigned integer
> + * Note that it will lose precision.
> + */
> +#define CVMX_SHOFT_TO_U64(m,
> e)                                                                   
>  \
> +	((((1ull << CVMX_SHOFT_MANT_BITS) | (m)) << (e)) >>
> CVMX_SHOFT_MANT_BITS)
> +
> +/**
> + * Convert to short-float from an unsigned integer
> + */
> +#define CVMX_SHOFT_FROM_U64(ui, m,
> e)                                                              \
> +	do
> {                                                                    
>                    \
> +		unsigned long long
> u;                                                              \
> +		unsigned int
> k;                                                                   
>  \
> +		k = (1ull << (CVMX_SHOFT_MANT_BITS + 1)) -
> 1;                                      \
> +		(e) =
> 0;                                                                   
>         \
> +		u = (ui) <<
> CVMX_SHOFT_MANT_BITS;                                                
>   \
> +		while ((u) > k)
> {                                                                  \
> +			u >>=
> 1;                                                                   
> \
> +			(e)++;                                         
>                             \
> +		}                                                      
>                             \
> +		(m) = u & (k >>
> 1);                                                                \
> +	} while (0);
> +
> +#define
> CVMX_SHOFT_MAX()                                                     
>                       \
> +	CVMX_SHOFT_TO_U64((1 << CVMX_SHOFT_MANT_BITS) - 1, (1 <<
> CVMX_SHOFT_EXP_BITS) - 1)
> +#define CVMX_SHOFT_MIN() CVMX_SHOFT_TO_U64(0, 0)
> +
> +#endif /* __CVMX_PKO3_QUEUE_H__ */
> diff --git a/arch/mips/mach-octeon/include/mach/cvmx-pow.h
> b/arch/mips/mach-octeon/include/mach/cvmx-pow.h
> new file mode 100644
> index 000000000000..0680ca258f12
> --- /dev/null
> +++ b/arch/mips/mach-octeon/include/mach/cvmx-pow.h
> @@ -0,0 +1,2991 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +/*
> + * Copyright (C) 2020 Marvell International Ltd.
> + *
> + * Interface to the hardware Scheduling unit.
> + *
> + * New, starting with SDK 1.7.0, cvmx-pow supports a number of
> + * extended consistency checks. The define
> + * CVMX_ENABLE_POW_CHECKS controls the runtime insertion of POW
> + * internal state checks to find common programming errors. If
> + * CVMX_ENABLE_POW_CHECKS is not defined, checks are by default
> + * enabled. For example, cvmx-pow will check for the following
> + * program errors or POW state inconsistency.
> + * - Requesting a POW operation with an active tag switch in
> + *   progress.
> + * - Waiting for a tag switch to complete for an excessively
> + *   long period. This is normally a sign of an error in locking
> + *   causing deadlock.
> + * - Illegal tag switches from NULL_NULL.
> + * - Illegal tag switches from NULL.
> + * - Illegal deschedule request.
> + * - WQE pointer not matching the one attached to the core by
> + *   the POW.
> + */
> +
> +#ifndef __CVMX_POW_H__
> +#define __CVMX_POW_H__
> +
> +#include "cvmx-wqe.h"
> +#include "cvmx-pow-defs.h"
> +#include "cvmx-sso-defs.h"
> +#include "cvmx-address.h"
> +#include "cvmx-coremask.h"
> +
> +/* Default to having all POW constancy checks turned on */
> +#ifndef CVMX_ENABLE_POW_CHECKS
> +#define CVMX_ENABLE_POW_CHECKS 1
> +#endif
> +
> +/*
> + * Special type for CN78XX style SSO groups (0..255),
> + * for distinction from legacy-style groups (0..15)
> + */
> +typedef union {
> +	u8 xgrp;
> +	/* Fields that map XGRP for backwards compatibility */
> +	struct __attribute__((__packed__)) {
> +		u8 group : 5;
> +		u8 qus : 3;
> +	};
> +} cvmx_xgrp_t;
> +
> +/*
> + * Softwsare-only structure to convey a return value
> + * containing multiple information fields about an work queue entry
> + */
> +typedef struct {
> +	u32 tag;
> +	u16 index;
> +	u8 grp; /* Legacy group # (0..15) */
> +	u8 tag_type;
> +} cvmx_pow_tag_info_t;
> +
> +/**
> + * Wait flag values for pow functions.
> + */
> +typedef enum {
> +	CVMX_POW_WAIT = 1,
> +	CVMX_POW_NO_WAIT = 0,
> +} cvmx_pow_wait_t;
> +
> +/**
> + *  POW tag operations.  These are used in the data stored to the
> POW.
> + */
> +typedef enum {
> +	CVMX_POW_TAG_OP_SWTAG = 0L,
> +	CVMX_POW_TAG_OP_SWTAG_FULL = 1L,
> +	CVMX_POW_TAG_OP_SWTAG_DESCH = 2L,
> +	CVMX_POW_TAG_OP_DESCH = 3L,
> +	CVMX_POW_TAG_OP_ADDWQ = 4L,
> +	CVMX_POW_TAG_OP_UPDATE_WQP_GRP = 5L,
> +	CVMX_POW_TAG_OP_SET_NSCHED = 6L,
> +	CVMX_POW_TAG_OP_CLR_NSCHED = 7L,
> +	CVMX_POW_TAG_OP_NOP = 15L
> +} cvmx_pow_tag_op_t;
> +
> +/**
> + * This structure defines the store data on a store to POW
> + */
> +typedef union {
> +	u64 u64;
> +	struct {
> +		u64 no_sched : 1;
> +		u64 unused : 2;
> +		u64 index : 13;
> +		cvmx_pow_tag_op_t op : 4;
> +		u64 unused2 : 2;
> +		u64 qos : 3;
> +		u64 grp : 4;
> +		cvmx_pow_tag_type_t type : 3;
> +		u64 tag : 32;
> +	} s_cn38xx;
> +	struct {
> +		u64 no_sched : 1;
> +		cvmx_pow_tag_op_t op : 4;
> +		u64 unused1 : 4;
> +		u64 index : 11;
> +		u64 unused2 : 1;
> +		u64 grp : 6;
> +		u64 unused3 : 3;
> +		cvmx_pow_tag_type_t type : 2;
> +		u64 tag : 32;
> +	} s_cn68xx_clr;
> +	struct {
> +		u64 no_sched : 1;
> +		cvmx_pow_tag_op_t op : 4;
> +		u64 unused1 : 12;
> +		u64 qos : 3;
> +		u64 unused2 : 1;
> +		u64 grp : 6;
> +		u64 unused3 : 3;
> +		cvmx_pow_tag_type_t type : 2;
> +		u64 tag : 32;
> +	} s_cn68xx_add;
> +	struct {
> +		u64 no_sched : 1;
> +		cvmx_pow_tag_op_t op : 4;
> +		u64 unused1 : 16;
> +		u64 grp : 6;
> +		u64 unused3 : 3;
> +		cvmx_pow_tag_type_t type : 2;
> +		u64 tag : 32;
> +	} s_cn68xx_other;
> +	struct {
> +		u64 rsvd_62_63 : 2;
> +		u64 grp : 10;
> +		cvmx_pow_tag_type_t type : 2;
> +		u64 no_sched : 1;
> +		u64 rsvd_48 : 1;
> +		cvmx_pow_tag_op_t op : 4;
> +		u64 rsvd_42_43 : 2;
> +		u64 wqp : 42;
> +	} s_cn78xx_other;
> +
> +} cvmx_pow_tag_req_t;
> +
> +union cvmx_pow_tag_req_addr {
> +	u64 u64;
> +	struct {
> +		u64 mem_region : 2;
> +		u64 reserved_49_61 : 13;
> +		u64 is_io : 1;
> +		u64 did : 8;
> +		u64 addr : 40;
> +	} s;
> +	struct {
> +		u64 mem_region : 2;
> +		u64 reserved_49_61 : 13;
> +		u64 is_io : 1;
> +		u64 did : 8;
> +		u64 node : 4;
> +		u64 tag : 32;
> +		u64 reserved_0_3 : 4;
> +	} s_cn78xx;
> +};
> +
> +/**
> + * This structure describes the address to load stuff from POW
> + */
> +typedef union {
> +	u64 u64;
> +	/**
> +	 * Address for new work request loads (did<2:0> == 0)
> +	 */
> +	struct {
> +		u64 mem_region : 2;
> +		u64 reserved_49_61 : 13;
> +		u64 is_io : 1;
> +		u64 did : 8;
> +		u64 reserved_4_39 : 36;
> +		u64 wait : 1;
> +		u64 reserved_0_2 : 3;
> +	} swork;
> +	struct {
> +		u64 mem_region : 2;
> +		u64 reserved_49_61 : 13;
> +		u64 is_io : 1;
> +		u64 did : 8;
> +		u64 node : 4;
> +		u64 reserved_32_35 : 4;
> +		u64 indexed : 1;
> +		u64 grouped : 1;
> +		u64 rtngrp : 1;
> +		u64 reserved_16_28 : 13;
> +		u64 index : 12;
> +		u64 wait : 1;
> +		u64 reserved_0_2 : 3;
> +	} swork_78xx;
> +	/**
> +	 * Address for loads to get POW internal status
> +	 */
> +	struct {
> +		u64 mem_region : 2;
> +		u64 reserved_49_61 : 13;
> +		u64 is_io : 1;
> +		u64 did : 8;
> +		u64 reserved_10_39 : 30;
> +		u64 coreid : 4;
> +		u64 get_rev : 1;
> +		u64 get_cur : 1;
> +		u64 get_wqp : 1;
> +		u64 reserved_0_2 : 3;
> +	} sstatus;
> +	/**
> +	 * Address for loads to get 68XX SS0 internal status
> +	 */
> +	struct {
> +		u64 mem_region : 2;
> +		u64 reserved_49_61 : 13;
> +		u64 is_io : 1;
> +		u64 did : 8;
> +		u64 reserved_14_39 : 26;
> +		u64 coreid : 5;
> +		u64 reserved_6_8 : 3;
> +		u64 opcode : 3;
> +		u64 reserved_0_2 : 3;
> +	} sstatus_cn68xx;
> +	/**
> +	 * Address for memory loads to get POW internal state
> +	 */
> +	struct {
> +		u64 mem_region : 2;
> +		u64 reserved_49_61 : 13;
> +		u64 is_io : 1;
> +		u64 did : 8;
> +		u64 reserved_16_39 : 24;
> +		u64 index : 11;
> +		u64 get_des : 1;
> +		u64 get_wqp : 1;
> +		u64 reserved_0_2 : 3;
> +	} smemload;
> +	/**
> +	 * Address for memory loads to get SSO internal state
> +	 */
> +	struct {
> +		u64 mem_region : 2;
> +		u64 reserved_49_61 : 13;
> +		u64 is_io : 1;
> +		u64 did : 8;
> +		u64 reserved_20_39 : 20;
> +		u64 index : 11;
> +		u64 reserved_6_8 : 3;
> +		u64 opcode : 3;
> +		u64 reserved_0_2 : 3;
> +	} smemload_cn68xx;
> +	/**
> +	 * Address for index/pointer loads
> +	 */
> +	struct {
> +		u64 mem_region : 2;
> +		u64 reserved_49_61 : 13;
> +		u64 is_io : 1;
> +		u64 did : 8;
> +		u64 reserved_9_39 : 31;
> +		u64 qosgrp : 4;
> +		u64 get_des_get_tail : 1;
> +		u64 get_rmt : 1;
> +		u64 reserved_0_2 : 3;
> +	} sindexload;
> +	/**
> +	 * Address for a Index/Pointer loads to get SSO internal state
> +	 */
> +	struct {
> +		u64 mem_region : 2;
> +		u64 reserved_49_61 : 13;
> +		u64 is_io : 1;
> +		u64 did : 8;
> +		u64 reserved_15_39 : 25;
> +		u64 qos_grp : 6;
> +		u64 reserved_6_8 : 3;
> +		u64 opcode : 3;
> +		u64 reserved_0_2 : 3;
> +	} sindexload_cn68xx;
> +	/**
> +	 * Address for NULL_RD request (did<2:0> == 4)
> +	 * when this is read, HW attempts to change the state to NULL
> if it is NULL_NULL
> +	 * (the hardware cannot switch from NULL_NULL to NULL if a POW
> entry is not available -
> +	 * software may need to recover by finishing another piece of
> work before a POW
> +	 * entry can ever become available.)
> +	 */
> +	struct {
> +		u64 mem_region : 2;
> +		u64 reserved_49_61 : 13;
> +		u64 is_io : 1;
> +		u64 did : 8;
> +		u64 reserved_0_39 : 40;
> +	} snull_rd;
> +} cvmx_pow_load_addr_t;
> +
> +/**
> + * This structure defines the response to a load/SENDSINGLE to POW
> (except CSR reads)
> + */
> +typedef union {
> +	u64 u64;
> +	/**
> +	 * Response to new work request loads
> +	 */
> +	struct {
> +		u64 no_work : 1;
> +		u64 pend_switch : 1;
> +		u64 tt : 2;
> +		u64 reserved_58_59 : 2;
> +		u64 grp : 10;
> +		u64 reserved_42_47 : 6;
> +		u64 addr : 42;
> +	} s_work;
> +
> +	/**
> +	 * Result for a POW Status Load (when get_cur==0 and
> get_wqp==0)
> +	 */
> +	struct {
> +		u64 reserved_62_63 : 2;
> +		u64 pend_switch : 1;
> +		u64 pend_switch_full : 1;
> +		u64 pend_switch_null : 1;
> +		u64 pend_desched : 1;
> +		u64 pend_desched_switch : 1;
> +		u64 pend_nosched : 1;
> +		u64 pend_new_work : 1;
> +		u64 pend_new_work_wait : 1;
> +		u64 pend_null_rd : 1;
> +		u64 pend_nosched_clr : 1;
> +		u64 reserved_51 : 1;
> +		u64 pend_index : 11;
> +		u64 pend_grp : 4;
> +		u64 reserved_34_35 : 2;
> +		u64 pend_type : 2;
> +		u64 pend_tag : 32;
> +	} s_sstatus0;
> +	/**
> +	 * Result for a SSO Status Load (when opcode is SL_PENDTAG)
> +	 */
> +	struct {
> +		u64 pend_switch : 1;
> +		u64 pend_get_work : 1;
> +		u64 pend_get_work_wait : 1;
> +		u64 pend_nosched : 1;
> +		u64 pend_nosched_clr : 1;
> +		u64 pend_desched : 1;
> +		u64 pend_alloc_we : 1;
> +		u64 reserved_48_56 : 9;
> +		u64 pend_index : 11;
> +		u64 reserved_34_36 : 3;
> +		u64 pend_type : 2;
> +		u64 pend_tag : 32;
> +	} s_sstatus0_cn68xx;
> +	/**
> +	 * Result for a POW Status Load (when get_cur==0 and
> get_wqp==1)
> +	 */
> +	struct {
> +		u64 reserved_62_63 : 2;
> +		u64 pend_switch : 1;
> +		u64 pend_switch_full : 1;
> +		u64 pend_switch_null : 1;
> +		u64 pend_desched : 1;
> +		u64 pend_desched_switch : 1;
> +		u64 pend_nosched : 1;
> +		u64 pend_new_work : 1;
> +		u64 pend_new_work_wait : 1;
> +		u64 pend_null_rd : 1;
> +		u64 pend_nosched_clr : 1;
> +		u64 reserved_51 : 1;
> +		u64 pend_index : 11;
> +		u64 pend_grp : 4;
> +		u64 pend_wqp : 36;
> +	} s_sstatus1;
> +	/**
> +	 * Result for a SSO Status Load (when opcode is SL_PENDWQP)
> +	 */
> +	struct {
> +		u64 pend_switch : 1;
> +		u64 pend_get_work : 1;
> +		u64 pend_get_work_wait : 1;
> +		u64 pend_nosched : 1;
> +		u64 pend_nosched_clr : 1;
> +		u64 pend_desched : 1;
> +		u64 pend_alloc_we : 1;
> +		u64 reserved_51_56 : 6;
> +		u64 pend_index : 11;
> +		u64 reserved_38_39 : 2;
> +		u64 pend_wqp : 38;
> +	} s_sstatus1_cn68xx;
> +
> +	struct {
> +		u64 pend_switch : 1;
> +		u64 pend_get_work : 1;
> +		u64 pend_get_work_wait : 1;
> +		u64 pend_nosched : 1;
> +		u64 pend_nosched_clr : 1;
> +		u64 pend_desched : 1;
> +		u64 pend_alloc_we : 1;
> +		u64 reserved_56 : 1;
> +		u64 prep_index : 12;
> +		u64 reserved_42_43 : 2;
> +		u64 pend_tag : 42;
> +	} s_sso_ppx_pendwqp_cn78xx;
> +	/**
> +	 * Result for a POW Status Load (when get_cur==1, get_wqp==0,
> and get_rev==0)
> +	 */
> +	struct {
> +		u64 reserved_62_63 : 2;
> +		u64 link_index : 11;
> +		u64 index : 11;
> +		u64 grp : 4;
> +		u64 head : 1;
> +		u64 tail : 1;
> +		u64 tag_type : 2;
> +		u64 tag : 32;
> +	} s_sstatus2;
> +	/**
> +	 * Result for a SSO Status Load (when opcode is SL_TAG)
> +	 */
> +	struct {
> +		u64 reserved_57_63 : 7;
> +		u64 index : 11;
> +		u64 reserved_45 : 1;
> +		u64 grp : 6;
> +		u64 head : 1;
> +		u64 tail : 1;
> +		u64 reserved_34_36 : 3;
> +		u64 tag_type : 2;
> +		u64 tag : 32;
> +	} s_sstatus2_cn68xx;
> +
> +	struct {
> +		u64 tailc : 1;
> +		u64 reserved_60_62 : 3;
> +		u64 index : 12;
> +		u64 reserved_46_47 : 2;
> +		u64 grp : 10;
> +		u64 head : 1;
> +		u64 tail : 1;
> +		u64 tt : 2;
> +		u64 tag : 32;
> +	} s_sso_ppx_tag_cn78xx;
> +	/**
> +	 * Result for a POW Status Load (when get_cur==1, get_wqp==0,
> and get_rev==1)
> +	 */
> +	struct {
> +		u64 reserved_62_63 : 2;
> +		u64 revlink_index : 11;
> +		u64 index : 11;
> +		u64 grp : 4;
> +		u64 head : 1;
> +		u64 tail : 1;
> +		u64 tag_type : 2;
> +		u64 tag : 32;
> +	} s_sstatus3;
> +	/**
> +	 * Result for a SSO Status Load (when opcode is SL_WQP)
> +	 */
> +	struct {
> +		u64 reserved_58_63 : 6;
> +		u64 index : 11;
> +		u64 reserved_46 : 1;
> +		u64 grp : 6;
> +		u64 reserved_38_39 : 2;
> +		u64 wqp : 38;
> +	} s_sstatus3_cn68xx;
> +
> +	struct {
> +		u64 reserved_58_63 : 6;
> +		u64 grp : 10;
> +		u64 reserved_42_47 : 6;
> +		u64 tag : 42;
> +	} s_sso_ppx_wqp_cn78xx;
> +	/**
> +	 * Result for a POW Status Load (when get_cur==1, get_wqp==1,
> and get_rev==0)
> +	 */
> +	struct {
> +		u64 reserved_62_63 : 2;
> +		u64 link_index : 11;
> +		u64 index : 11;
> +		u64 grp : 4;
> +		u64 wqp : 36;
> +	} s_sstatus4;
> +	/**
> +	 * Result for a SSO Status Load (when opcode is SL_LINKS)
> +	 */
> +	struct {
> +		u64 reserved_46_63 : 18;
> +		u64 index : 11;
> +		u64 reserved_34 : 1;
> +		u64 grp : 6;
> +		u64 head : 1;
> +		u64 tail : 1;
> +		u64 reserved_24_25 : 2;
> +		u64 revlink_index : 11;
> +		u64 reserved_11_12 : 2;
> +		u64 link_index : 11;
> +	} s_sstatus4_cn68xx;
> +
> +	struct {
> +		u64 tailc : 1;
> +		u64 reserved_60_62 : 3;
> +		u64 index : 12;
> +		u64 reserved_38_47 : 10;
> +		u64 grp : 10;
> +		u64 head : 1;
> +		u64 tail : 1;
> +		u64 reserved_25 : 1;
> +		u64 revlink_index : 12;
> +		u64 link_index_vld : 1;
> +		u64 link_index : 12;
> +	} s_sso_ppx_links_cn78xx;
> +	/**
> +	 * Result for a POW Status Load (when get_cur==1, get_wqp==1,
> and get_rev==1)
> +	 */
> +	struct {
> +		u64 reserved_62_63 : 2;
> +		u64 revlink_index : 11;
> +		u64 index : 11;
> +		u64 grp : 4;
> +		u64 wqp : 36;
> +	} s_sstatus5;
> +	/**
> +	 * Result For POW Memory Load (get_des == 0 and get_wqp == 0)
> +	 */
> +	struct {
> +		u64 reserved_51_63 : 13;
> +		u64 next_index : 11;
> +		u64 grp : 4;
> +		u64 reserved_35 : 1;
> +		u64 tail : 1;
> +		u64 tag_type : 2;
> +		u64 tag : 32;
> +	} s_smemload0;
> +	/**
> +	 * Result For SSO Memory Load (opcode is ML_TAG)
> +	 */
> +	struct {
> +		u64 reserved_38_63 : 26;
> +		u64 tail : 1;
> +		u64 reserved_34_36 : 3;
> +		u64 tag_type : 2;
> +		u64 tag : 32;
> +	} s_smemload0_cn68xx;
> +
> +	struct {
> +		u64 reserved_39_63 : 25;
> +		u64 tail : 1;
> +		u64 reserved_34_36 : 3;
> +		u64 tag_type : 2;
> +		u64 tag : 32;
> +	} s_sso_iaq_ppx_tag_cn78xx;
> +	/**
> +	 * Result For POW Memory Load (get_des == 0 and get_wqp == 1)
> +	 */
> +	struct {
> +		u64 reserved_51_63 : 13;
> +		u64 next_index : 11;
> +		u64 grp : 4;
> +		u64 wqp : 36;
> +	} s_smemload1;
> +	/**
> +	 * Result For SSO Memory Load (opcode is ML_WQPGRP)
> +	 */
> +	struct {
> +		u64 reserved_48_63 : 16;
> +		u64 nosched : 1;
> +		u64 reserved_46 : 1;
> +		u64 grp : 6;
> +		u64 reserved_38_39 : 2;
> +		u64 wqp : 38;
> +	} s_smemload1_cn68xx;
> +
> +	/**
> +	 * Entry structures for the CN7XXX chips.
> +	 */
> +	struct {
> +		u64 reserved_39_63 : 25;
> +		u64 tailc : 1;
> +		u64 tail : 1;
> +		u64 reserved_34_36 : 3;
> +		u64 tt : 2;
> +		u64 tag : 32;
> +	} s_sso_ientx_tag_cn78xx;
> +
> +	struct {
> +		u64 reserved_62_63 : 2;
> +		u64 head : 1;
> +		u64 nosched : 1;
> +		u64 reserved_56_59 : 4;
> +		u64 grp : 8;
> +		u64 reserved_42_47 : 6;
> +		u64 wqp : 42;
> +	} s_sso_ientx_wqpgrp_cn73xx;
> +
> +	struct {
> +		u64 reserved_62_63 : 2;
> +		u64 head : 1;
> +		u64 nosched : 1;
> +		u64 reserved_58_59 : 2;
> +		u64 grp : 10;
> +		u64 reserved_42_47 : 6;
> +		u64 wqp : 42;
> +	} s_sso_ientx_wqpgrp_cn78xx;
> +
> +	struct {
> +		u64 reserved_38_63 : 26;
> +		u64 pend_switch : 1;
> +		u64 reserved_34_36 : 3;
> +		u64 pend_tt : 2;
> +		u64 pend_tag : 32;
> +	} s_sso_ientx_pendtag_cn78xx;
> +
> +	struct {
> +		u64 reserved_26_63 : 38;
> +		u64 prev_index : 10;
> +		u64 reserved_11_15 : 5;
> +		u64 next_index_vld : 1;
> +		u64 next_index : 10;
> +	} s_sso_ientx_links_cn73xx;
> +
> +	struct {
> +		u64 reserved_28_63 : 36;
> +		u64 prev_index : 12;
> +		u64 reserved_13_15 : 3;
> +		u64 next_index_vld : 1;
> +		u64 next_index : 12;
> +	} s_sso_ientx_links_cn78xx;
> +
> +	/**
> +	 * Result For POW Memory Load (get_des == 1)
> +	 */
> +	struct {
> +		u64 reserved_51_63 : 13;
> +		u64 fwd_index : 11;
> +		u64 grp : 4;
> +		u64 nosched : 1;
> +		u64 pend_switch : 1;
> +		u64 pend_type : 2;
> +		u64 pend_tag : 32;
> +	} s_smemload2;
> +	/**
> +	 * Result For SSO Memory Load (opcode is ML_PENTAG)
> +	 */
> +	struct {
> +		u64 reserved_38_63 : 26;
> +		u64 pend_switch : 1;
> +		u64 reserved_34_36 : 3;
> +		u64 pend_type : 2;
> +		u64 pend_tag : 32;
> +	} s_smemload2_cn68xx;
> +
> +	struct {
> +		u64 pend_switch : 1;
> +		u64 pend_get_work : 1;
> +		u64 pend_get_work_wait : 1;
> +		u64 pend_nosched : 1;
> +		u64 pend_nosched_clr : 1;
> +		u64 pend_desched : 1;
> +		u64 pend_alloc_we : 1;
> +		u64 reserved_34_56 : 23;
> +		u64 pend_tt : 2;
> +		u64 pend_tag : 32;
> +	} s_sso_ppx_pendtag_cn78xx;
> +	/**
> +	 * Result For SSO Memory Load (opcode is ML_LINKS)
> +	 */
> +	struct {
> +		u64 reserved_24_63 : 40;
> +		u64 fwd_index : 11;
> +		u64 reserved_11_12 : 2;
> +		u64 next_index : 11;
> +	} s_smemload3_cn68xx;
> +
> +	/**
> +	 * Result For POW Index/Pointer Load (get_rmt ==
> 0/get_des_get_tail == 0)
> +	 */
> +	struct {
> +		u64 reserved_52_63 : 12;
> +		u64 free_val : 1;
> +		u64 free_one : 1;
> +		u64 reserved_49 : 1;
> +		u64 free_head : 11;
> +		u64 reserved_37 : 1;
> +		u64 free_tail : 11;
> +		u64 loc_val : 1;
> +		u64 loc_one : 1;
> +		u64 reserved_23 : 1;
> +		u64 loc_head : 11;
> +		u64 reserved_11 : 1;
> +		u64 loc_tail : 11;
> +	} sindexload0;
> +	/**
> +	 * Result for SSO Index/Pointer Load(opcode ==
> +	 * IPL_IQ/IPL_DESCHED/IPL_NOSCHED)
> +	 */
> +	struct {
> +		u64 reserved_28_63 : 36;
> +		u64 queue_val : 1;
> +		u64 queue_one : 1;
> +		u64 reserved_24_25 : 2;
> +		u64 queue_head : 11;
> +		u64 reserved_11_12 : 2;
> +		u64 queue_tail : 11;
> +	} sindexload0_cn68xx;
> +	/**
> +	 * Result For POW Index/Pointer Load (get_rmt ==
> 0/get_des_get_tail == 1)
> +	 */
> +	struct {
> +		u64 reserved_52_63 : 12;
> +		u64 nosched_val : 1;
> +		u64 nosched_one : 1;
> +		u64 reserved_49 : 1;
> +		u64 nosched_head : 11;
> +		u64 reserved_37 : 1;
> +		u64 nosched_tail : 11;
> +		u64 des_val : 1;
> +		u64 des_one : 1;
> +		u64 reserved_23 : 1;
> +		u64 des_head : 11;
> +		u64 reserved_11 : 1;
> +		u64 des_tail : 11;
> +	} sindexload1;
> +	/**
> +	 * Result for SSO Index/Pointer Load(opcode ==
> IPL_FREE0/IPL_FREE1/IPL_FREE2)
> +	 */
> +	struct {
> +		u64 reserved_60_63 : 4;
> +		u64 qnum_head : 2;
> +		u64 qnum_tail : 2;
> +		u64 reserved_28_55 : 28;
> +		u64 queue_val : 1;
> +		u64 queue_one : 1;
> +		u64 reserved_24_25 : 2;
> +		u64 queue_head : 11;
> +		u64 reserved_11_12 : 2;
> +		u64 queue_tail : 11;
> +	} sindexload1_cn68xx;
> +	/**
> +	 * Result For POW Index/Pointer Load (get_rmt ==
> 1/get_des_get_tail == 0)
> +	 */
> +	struct {
> +		u64 reserved_39_63 : 25;
> +		u64 rmt_is_head : 1;
> +		u64 rmt_val : 1;
> +		u64 rmt_one : 1;
> +		u64 rmt_head : 36;
> +	} sindexload2;
> +	/**
> +	 * Result For POW Index/Pointer Load (get_rmt ==
> 1/get_des_get_tail == 1)
> +	 */
> +	struct {
> +		u64 reserved_39_63 : 25;
> +		u64 rmt_is_head : 1;
> +		u64 rmt_val : 1;
> +		u64 rmt_one : 1;
> +		u64 rmt_tail : 36;
> +	} sindexload3;
> +	/**
> +	 * Response to NULL_RD request loads
> +	 */
> +	struct {
> +		u64 unused : 62;
> +		u64 state : 2;
> +	} s_null_rd;
> +
> +} cvmx_pow_tag_load_resp_t;
> +
> +typedef union {
> +	u64 u64;
> +	struct {
> +		u64 reserved_57_63 : 7;
> +		u64 index : 11;
> +		u64 reserved_45 : 1;
> +		u64 grp : 6;
> +		u64 head : 1;
> +		u64 tail : 1;
> +		u64 reserved_34_36 : 3;
> +		u64 tag_type : 2;
> +		u64 tag : 32;
> +	} s;
> +} cvmx_pow_sl_tag_resp_t;
> +
> +/**
> + * This structure describes the address used for stores to the POW.
> + *  The store address is meaningful on stores to the POW.  The
> hardware assumes that an aligned
> + *  64-bit store was used for all these stores.
> + *  Note the assumption that the work queue entry is aligned on an
> 8-byte
> + *  boundary (since the low-order 3 address bits must be zero).
> + *  Note that not all fields are used by all operations.
> + *
> + *  NOTE: The following is the behavior of the pending switch bit at
> the PP
> + *       for POW stores (i.e. when did<7:3> == 0xc)
> + *     - did<2:0> == 0      => pending switch bit is set
> + *     - did<2:0> == 1      => no affect on the pending switch bit
> + *     - did<2:0> == 3      => pending switch bit is cleared
> + *     - did<2:0> == 7      => no affect on the pending switch bit
> + *     - did<2:0> == others => must not be used
> + *     - No other loads/stores have an affect on the pending switch
> bit
> + *     - The switch bus from POW can clear the pending switch bit
> + *
> + *  NOTE: did<2:0> == 2 is used by the HW for a special single-cycle 
> ADDWQ command
> + *  that only contains the pointer). SW must never use did<2:0> ==
> 2.
> + */
> +typedef union {
> +	u64 u64;
> +	struct {
> +		u64 mem_reg : 2;
> +		u64 reserved_49_61 : 13;
> +		u64 is_io : 1;
> +		u64 did : 8;
> +		u64 addr : 40;
> +	} stag;
> +} cvmx_pow_tag_store_addr_t; /* FIXME- this type is unused */
> +
> +/**
> + * Decode of the store data when an IOBDMA SENDSINGLE is sent to POW
> + */
> +typedef union {
> +	u64 u64;
> +	struct {
> +		u64 scraddr : 8;
> +		u64 len : 8;
> +		u64 did : 8;
> +		u64 unused : 36;
> +		u64 wait : 1;
> +		u64 unused2 : 3;
> +	} s;
> +	struct {
> +		u64 scraddr : 8;
> +		u64 len : 8;
> +		u64 did : 8;
> +		u64 node : 4;
> +		u64 unused1 : 4;
> +		u64 indexed : 1;
> +		u64 grouped : 1;
> +		u64 rtngrp : 1;
> +		u64 unused2 : 13;
> +		u64 index_grp_mask : 12;
> +		u64 wait : 1;
> +		u64 unused3 : 3;
> +	} s_cn78xx;
> +} cvmx_pow_iobdma_store_t;
> +
> +/* CSR typedefs have been moved to cvmx-pow-defs.h */
> +
> +/*enum for group priority parameters which needs modification*/
> +enum cvmx_sso_group_modify_mask {
> +	CVMX_SSO_MODIFY_GROUP_PRIORITY = 0x01,
> +	CVMX_SSO_MODIFY_GROUP_WEIGHT = 0x02,
> +	CVMX_SSO_MODIFY_GROUP_AFFINITY = 0x04
> +};
> +
> +/**
> + * @INTERNAL
> + * Return the number of SSO groups for a given SoC model
> + */
> +static inline unsigned int cvmx_sso_num_xgrp(void)
> +{
> +	if (OCTEON_IS_MODEL(OCTEON_CN78XX))
> +		return 256;
> +	if (OCTEON_IS_MODEL(OCTEON_CNF75XX))
> +		return 64;
> +	if (OCTEON_IS_MODEL(OCTEON_CN73XX))
> +		return 64;
> +	printf("ERROR: %s: Unknown model\n", __func__);
> +	return 0;
> +}
> +
> +/**
> + * @INTERNAL
> + * Return the number of POW groups on current model.
> + * In case of CN78XX/CN73XX this is the number of equivalent
> + * "legacy groups" on the chip when it is used in backward
> + * compatible mode.
> + */
> +static inline unsigned int cvmx_pow_num_groups(void)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE))
> +		return cvmx_sso_num_xgrp() >> 3;
> +	else if (octeon_has_feature(OCTEON_FEATURE_CN68XX_WQE))
> +		return 64;
> +	else
> +		return 16;
> +}
> +
> +/**
> + * @INTERNAL
> + * Return the number of mask-set registers.
> + */
> +static inline unsigned int cvmx_sso_num_maskset(void)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE))
> +		return 2;
> +	else
> +		return 1;
> +}
> +
> +/**
> + * Get the POW tag for this core. This returns the current
> + * tag type, tag, group, and POW entry index associated with
> + * this core. Index is only valid if the tag type isn't NULL_NULL.
> + * If a tag switch is pending this routine returns the tag before
> + * the tag switch, not after.
> + *
> + * @return Current tag
> + */
> +static inline cvmx_pow_tag_info_t cvmx_pow_get_current_tag(void)
> +{
> +	cvmx_pow_load_addr_t load_addr;
> +	cvmx_pow_tag_info_t result;
> +
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		cvmx_sso_sl_ppx_tag_t sl_ppx_tag;
> +		cvmx_xgrp_t xgrp;
> +		int node, core;
> +
> +		CVMX_SYNCS;
> +		node = cvmx_get_node_num();
> +		core = cvmx_get_local_core_num();
> +		sl_ppx_tag.u64 = csr_rd_node(node,
> CVMX_SSO_SL_PPX_TAG(core));
> +		result.index = sl_ppx_tag.s.index;
> +		result.tag_type = sl_ppx_tag.s.tt;
> +		result.tag = sl_ppx_tag.s.tag;
> +
> +		/* Get native XGRP value */
> +		xgrp.xgrp = sl_ppx_tag.s.grp;
> +
> +		/* Return legacy style group 0..15 */
> +		result.grp = xgrp.group;
> +	} else if (octeon_has_feature(OCTEON_FEATURE_CN68XX_WQE)) {
> +		cvmx_pow_sl_tag_resp_t load_resp;
> +
> +		load_addr.u64 = 0;
> +		load_addr.sstatus_cn68xx.mem_region = CVMX_IO_SEG;
> +		load_addr.sstatus_cn68xx.is_io = 1;
> +		load_addr.sstatus_cn68xx.did = CVMX_OCT_DID_TAG_TAG5;
> +		load_addr.sstatus_cn68xx.coreid = cvmx_get_core_num();
> +		load_addr.sstatus_cn68xx.opcode = 3;
> +		load_resp.u64 = csr_rd(load_addr.u64);
> +		result.grp = load_resp.s.grp;
> +		result.index = load_resp.s.index;
> +		result.tag_type = load_resp.s.tag_type;
> +		result.tag = load_resp.s.tag;
> +	} else {
> +		cvmx_pow_tag_load_resp_t load_resp;
> +
> +		load_addr.u64 = 0;
> +		load_addr.sstatus.mem_region = CVMX_IO_SEG;
> +		load_addr.sstatus.is_io = 1;
> +		load_addr.sstatus.did = CVMX_OCT_DID_TAG_TAG1;
> +		load_addr.sstatus.coreid = cvmx_get_core_num();
> +		load_addr.sstatus.get_cur = 1;
> +		load_resp.u64 = csr_rd(load_addr.u64);
> +		result.grp = load_resp.s_sstatus2.grp;
> +		result.index = load_resp.s_sstatus2.index;
> +		result.tag_type = load_resp.s_sstatus2.tag_type;
> +		result.tag = load_resp.s_sstatus2.tag;
> +	}
> +	return result;
> +}
> +
> +/**
> + * Get the POW WQE for this core. This returns the work queue
> + * entry currently associated with this core.
> + *
> + * @return WQE pointer
> + */
> +static inline cvmx_wqe_t *cvmx_pow_get_current_wqp(void)
> +{
> +	cvmx_pow_load_addr_t load_addr;
> +	cvmx_pow_tag_load_resp_t load_resp;
> +
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		cvmx_sso_sl_ppx_wqp_t sso_wqp;
> +		int node = cvmx_get_node_num();
> +		int core = cvmx_get_local_core_num();
> +
> +		sso_wqp.u64 = csr_rd_node(node,
> CVMX_SSO_SL_PPX_WQP(core));
> +		if (sso_wqp.s.wqp)
> +			return (cvmx_wqe_t
> *)cvmx_phys_to_ptr(sso_wqp.s.wqp);
> +		return (cvmx_wqe_t *)0;
> +	}
> +	if (octeon_has_feature(OCTEON_FEATURE_CN68XX_WQE)) {
> +		load_addr.u64 = 0;
> +		load_addr.sstatus_cn68xx.mem_region = CVMX_IO_SEG;
> +		load_addr.sstatus_cn68xx.is_io = 1;
> +		load_addr.sstatus_cn68xx.did = CVMX_OCT_DID_TAG_TAG5;
> +		load_addr.sstatus_cn68xx.coreid = cvmx_get_core_num();
> +		load_addr.sstatus_cn68xx.opcode = 4;
> +		load_resp.u64 = csr_rd(load_addr.u64);
> +		if (load_resp.s_sstatus3_cn68xx.wqp)
> +			return (cvmx_wqe_t
> *)cvmx_phys_to_ptr(load_resp.s_sstatus3_cn68xx.wqp);
> +		else
> +			return (cvmx_wqe_t *)0;
> +	} else {
> +		load_addr.u64 = 0;
> +		load_addr.sstatus.mem_region = CVMX_IO_SEG;
> +		load_addr.sstatus.is_io = 1;
> +		load_addr.sstatus.did = CVMX_OCT_DID_TAG_TAG1;
> +		load_addr.sstatus.coreid = cvmx_get_core_num();
> +		load_addr.sstatus.get_cur = 1;
> +		load_addr.sstatus.get_wqp = 1;
> +		load_resp.u64 = csr_rd(load_addr.u64);
> +		return (cvmx_wqe_t
> *)cvmx_phys_to_ptr(load_resp.s_sstatus4.wqp);
> +	}
> +}
> +
> +/**
> + * @INTERNAL
> + * Print a warning if a tag switch is pending for this core
> + *
> + * @param function Function name checking for a pending tag switch
> + */
> +static inline void __cvmx_pow_warn_if_pending_switch(const char
> *function)
> +{
> +	u64 switch_complete;
> +
> +	CVMX_MF_CHORD(switch_complete);
> +	cvmx_warn_if(!switch_complete, "%s called with tag switch in
> progress\n", function);
> +}
> +
> +/**
> + * Waits for a tag switch to complete by polling the completion bit.
> + * Note that switches to NULL complete immediately and do not need
> + * to be waited for.
> + */
> +static inline void cvmx_pow_tag_sw_wait(void)
> +{
> +	const u64 TIMEOUT_MS = 10; /* 10ms timeout */
> +	u64 switch_complete;
> +	u64 start_cycle;
> +
> +	if (CVMX_ENABLE_POW_CHECKS)
> +		start_cycle = get_timer(0);
> +
> +	while (1) {
> +		CVMX_MF_CHORD(switch_complete);
> +		if (cvmx_likely(switch_complete))
> +			break;
> +
> +		if (CVMX_ENABLE_POW_CHECKS) {
> +			if (cvmx_unlikely(get_timer(start_cycle) >
> TIMEOUT_MS)) {
> +				debug("WARNING: %s: Tag switch is
> taking a long time, possible deadlock\n",
> +				      __func__);
> +			}
> +		}
> +	}
> +}
> +
> +/**
> + * Synchronous work request.  Requests work from the POW.
> + * This function does NOT wait for previous tag switches to
> complete,
> + * so the caller must ensure that there is not a pending tag switch.
> + *
> + * @param wait   When set, call stalls until work becomes available,
> or
> + *               times out. If not set, returns immediately.
> + *
> + * @return Returns the WQE pointer from POW. Returns NULL if no work
> was
> + * available.
> + */
> +static inline cvmx_wqe_t
> *cvmx_pow_work_request_sync_nocheck(cvmx_pow_wait_t wait)
> +{
> +	cvmx_pow_load_addr_t ptr;
> +	cvmx_pow_tag_load_resp_t result;
> +
> +	if (CVMX_ENABLE_POW_CHECKS)
> +		__cvmx_pow_warn_if_pending_switch(__func__);
> +
> +	ptr.u64 = 0;
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		ptr.swork_78xx.node = cvmx_get_node_num();
> +		ptr.swork_78xx.mem_region = CVMX_IO_SEG;
> +		ptr.swork_78xx.is_io = 1;
> +		ptr.swork_78xx.did = CVMX_OCT_DID_TAG_SWTAG;
> +		ptr.swork_78xx.wait = wait;
> +	} else {
> +		ptr.swork.mem_region = CVMX_IO_SEG;
> +		ptr.swork.is_io = 1;
> +		ptr.swork.did = CVMX_OCT_DID_TAG_SWTAG;
> +		ptr.swork.wait = wait;
> +	}
> +
> +	result.u64 = csr_rd(ptr.u64);
> +	if (result.s_work.no_work)
> +		return NULL;
> +	else
> +		return (cvmx_wqe_t
> *)cvmx_phys_to_ptr(result.s_work.addr);
> +}
> +
> +/**
> + * Synchronous work request.  Requests work from the POW.
> + * This function waits for any previous tag switch to complete
> before
> + * requesting the new work.
> + *
> + * @param wait   When set, call stalls until work becomes available,
> or
> + *               times out. If not set, returns immediately.
> + *
> + * @return Returns the WQE pointer from POW. Returns NULL if no work
> was
> + * available.
> + */
> +static inline cvmx_wqe_t *cvmx_pow_work_request_sync(cvmx_pow_wait_t
> wait)
> +{
> +	/* Must not have a switch pending when requesting work */
> +	cvmx_pow_tag_sw_wait();
> +	return (cvmx_pow_work_request_sync_nocheck(wait));
> +}
> +
> +/**
> + * Synchronous null_rd request.  Requests a switch out of NULL_NULL
> POW state.
> + * This function waits for any previous tag switch to complete
> before
> + * requesting the null_rd.
> + *
> + * @return Returns the POW state of type cvmx_pow_tag_type_t.
> + */
> +static inline cvmx_pow_tag_type_t
> cvmx_pow_work_request_null_rd(void)
> +{
> +	cvmx_pow_load_addr_t ptr;
> +	cvmx_pow_tag_load_resp_t result;
> +
> +	/* Must not have a switch pending when requesting work */
> +	cvmx_pow_tag_sw_wait();
> +
> +	ptr.u64 = 0;
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		ptr.swork_78xx.mem_region = CVMX_IO_SEG;
> +		ptr.swork_78xx.is_io = 1;
> +		ptr.swork_78xx.did = CVMX_OCT_DID_TAG_NULL_RD;
> +		ptr.swork_78xx.node = cvmx_get_node_num();
> +	} else {
> +		ptr.snull_rd.mem_region = CVMX_IO_SEG;
> +		ptr.snull_rd.is_io = 1;
> +		ptr.snull_rd.did = CVMX_OCT_DID_TAG_NULL_RD;
> +	}
> +	result.u64 = csr_rd(ptr.u64);
> +	return (cvmx_pow_tag_type_t)result.s_null_rd.state;
> +}
> +
> +/**
> + * Asynchronous work request.
> + * Work is requested from the POW unit, and should later be checked
> with
> + * function cvmx_pow_work_response_async.
> + * This function does NOT wait for previous tag switches to
> complete,
> + * so the caller must ensure that there is not a pending tag switch.
> + *
> + * @param scr_addr Scratch memory address that response will be
> returned to,
> + *     which is either a valid WQE, or a response with the invalid
> bit set.
> + *     Byte address, must be 8 byte aligned.
> + * @param wait 1 to cause response to wait for work to become
> available
> + *               (or timeout)
> + *             0 to cause response to return immediately
> + */
> +static inline void cvmx_pow_work_request_async_nocheck(int scr_addr,
> cvmx_pow_wait_t wait)
> +{
> +	cvmx_pow_iobdma_store_t data;
> +
> +	if (CVMX_ENABLE_POW_CHECKS)
> +		__cvmx_pow_warn_if_pending_switch(__func__);
> +
> +	/* scr_addr must be 8 byte aligned */
> +	data.u64 = 0;
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		data.s_cn78xx.node = cvmx_get_node_num();
> +		data.s_cn78xx.scraddr = scr_addr >> 3;
> +		data.s_cn78xx.len = 1;
> +		data.s_cn78xx.did = CVMX_OCT_DID_TAG_SWTAG;
> +		data.s_cn78xx.wait = wait;
> +	} else {
> +		data.s.scraddr = scr_addr >> 3;
> +		data.s.len = 1;
> +		data.s.did = CVMX_OCT_DID_TAG_SWTAG;
> +		data.s.wait = wait;
> +	}
> +	cvmx_send_single(data.u64);
> +}
> +
> +/**
> + * Asynchronous work request.
> + * Work is requested from the POW unit, and should later be checked
> with
> + * function cvmx_pow_work_response_async.
> + * This function waits for any previous tag switch to complete
> before
> + * requesting the new work.
> + *
> + * @param scr_addr Scratch memory address that response will be
> returned to,
> + *     which is either a valid WQE, or a response with the invalid
> bit set.
> + *     Byte address, must be 8 byte aligned.
> + * @param wait 1 to cause response to wait for work to become
> available
> + *               (or timeout)
> + *             0 to cause response to return immediately
> + */
> +static inline void cvmx_pow_work_request_async(int scr_addr,
> cvmx_pow_wait_t wait)
> +{
> +	/* Must not have a switch pending when requesting work */
> +	cvmx_pow_tag_sw_wait();
> +	cvmx_pow_work_request_async_nocheck(scr_addr, wait);
> +}
> +
> +/**
> + * Gets result of asynchronous work request.  Performs a IOBDMA sync
> + * to wait for the response.
> + *
> + * @param scr_addr Scratch memory address to get result from
> + *                  Byte address, must be 8 byte aligned.
> + * @return Returns the WQE from the scratch register, or NULL if no
> work was
> + *         available.
> + */
> +static inline cvmx_wqe_t *cvmx_pow_work_response_async(int scr_addr)
> +{
> +	cvmx_pow_tag_load_resp_t result;
> +
> +	CVMX_SYNCIOBDMA;
> +	result.u64 = cvmx_scratch_read64(scr_addr);
> +	if (result.s_work.no_work)
> +		return NULL;
> +	else
> +		return (cvmx_wqe_t
> *)cvmx_phys_to_ptr(result.s_work.addr);
> +}
> +
> +/**
> + * Checks if a work queue entry pointer returned by a work
> + * request is valid.  It may be invalid due to no work
> + * being available or due to a timeout.
> + *
> + * @param wqe_ptr pointer to a work queue entry returned by the POW
> + *
> + * @return 0 if pointer is valid
> + *         1 if invalid (no work was returned)
> + */
> +static inline u64 cvmx_pow_work_invalid(cvmx_wqe_t *wqe_ptr)
> +{
> +	return (!wqe_ptr); /* FIXME: improve */
> +}
> +
> +/**
> + * Starts a tag switch to the provided tag value and tag
> type.  Completion for
> + * the tag switch must be checked for separately.
> + * This function does NOT update the
> + * work queue entry in dram to match tag value and type, so the
> application must
> + * keep track of these if they are important to the application.
> + * This tag switch command must not be used for switches to NULL, as
> the tag
> + * switch pending bit will be set by the switch request, but never
> cleared by
> + * the hardware.
> + *
> + * NOTE: This should not be used when switching from a NULL
> tag.  Use
> + * cvmx_pow_tag_sw_full() instead.
> + *
> + * This function does no checks, so the caller must ensure that any
> previous tag
> + * switch has completed.
> + *
> + * @param tag      new tag value
> + * @param tag_type new tag type (ordered or atomic)
> + */
> +static inline void cvmx_pow_tag_sw_nocheck(u32 tag,
> cvmx_pow_tag_type_t tag_type)
> +{
> +	union cvmx_pow_tag_req_addr ptr;
> +	cvmx_pow_tag_req_t tag_req;
> +
> +	if (CVMX_ENABLE_POW_CHECKS) {
> +		cvmx_pow_tag_info_t current_tag;
> +
> +		__cvmx_pow_warn_if_pending_switch(__func__);
> +		current_tag = cvmx_pow_get_current_tag();
> +		cvmx_warn_if(current_tag.tag_type ==
> CVMX_POW_TAG_TYPE_NULL_NULL,
> +			     "%s called with NULL_NULL tag\n",
> __func__);
> +		cvmx_warn_if(current_tag.tag_type ==
> CVMX_POW_TAG_TYPE_NULL,
> +			     "%s called with NULL tag\n", __func__);
> +		cvmx_warn_if((current_tag.tag_type == tag_type) &&
> (current_tag.tag == tag),
> +			     "%s called to perform a tag switch to the
> same tag\n", __func__);
> +		cvmx_warn_if(
> +			tag_type == CVMX_POW_TAG_TYPE_NULL,
> +			"%s called to perform a tag switch to NULL. Use
> cvmx_pow_tag_sw_null() instead\n",
> +			__func__);
> +	}
> +
> +	/*
> +	 * Note that WQE in DRAM is not updated here, as the POW does
> not read
> +	 * from DRAM once the WQE is in flight.  See hardware manual
> for
> +	 * complete details.
> +	 * It is the application's responsibility to keep track of the
> +	 * current tag value if that is important.
> +	 */
> +	tag_req.u64 = 0;
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		tag_req.s_cn78xx_other.op = CVMX_POW_TAG_OP_SWTAG;
> +		tag_req.s_cn78xx_other.type = tag_type;
> +	} else if (octeon_has_feature(OCTEON_FEATURE_CN68XX_WQE)) {
> +		tag_req.s_cn68xx_other.op = CVMX_POW_TAG_OP_SWTAG;
> +		tag_req.s_cn68xx_other.tag = tag;
> +		tag_req.s_cn68xx_other.type = tag_type;
> +	} else {
> +		tag_req.s_cn38xx.op = CVMX_POW_TAG_OP_SWTAG;
> +		tag_req.s_cn38xx.tag = tag;
> +		tag_req.s_cn38xx.type = tag_type;
> +	}
> +	ptr.u64 = 0;
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		ptr.s_cn78xx.mem_region = CVMX_IO_SEG;
> +		ptr.s_cn78xx.is_io = 1;
> +		ptr.s_cn78xx.did = CVMX_OCT_DID_TAG_SWTAG;
> +		ptr.s_cn78xx.node = cvmx_get_node_num();
> +		ptr.s_cn78xx.tag = tag;
> +	} else {
> +		ptr.s.mem_region = CVMX_IO_SEG;
> +		ptr.s.is_io = 1;
> +		ptr.s.did = CVMX_OCT_DID_TAG_SWTAG;
> +	}
> +	/* Once this store arrives at POW, it will attempt the switch
> +	   software must wait for the switch to complete separately */
> +	cvmx_write_io(ptr.u64, tag_req.u64);
> +}
> +
> +/**
> + * Starts a tag switch to the provided tag value and tag
> type.  Completion for
> + * the tag switch must be checked for separately.
> + * This function does NOT update the
> + * work queue entry in dram to match tag value and type, so the
> application must
> + * keep track of these if they are important to the application.
> + * This tag switch command must not be used for switches to NULL, as
> the tag
> + * switch pending bit will be set by the switch request, but never
> cleared by
> + * the hardware.
> + *
> + * NOTE: This should not be used when switching from a NULL
> tag.  Use
> + * cvmx_pow_tag_sw_full() instead.
> + *
> + * This function waits for any previous tag switch to complete, and
> also
> + * displays an error on tag switches to NULL.
> + *
> + * @param tag      new tag value
> + * @param tag_type new tag type (ordered or atomic)
> + */
> +static inline void cvmx_pow_tag_sw(u32 tag, cvmx_pow_tag_type_t
> tag_type)
> +{
> +	/*
> +	 * Note that WQE in DRAM is not updated here, as the POW does
> not read
> +	 * from DRAM once the WQE is in flight.  See hardware manual
> for
> +	 * complete details. It is the application's responsibility to
> keep
> +	 * track of the current tag value if that is important.
> +	 */
> +
> +	/*
> +	 * Ensure that there is not a pending tag switch, as a tag
> switch
> +	 * cannot be started if a previous switch is still pending.
> +	 */
> +	cvmx_pow_tag_sw_wait();
> +	cvmx_pow_tag_sw_nocheck(tag, tag_type);
> +}
> +
> +/**
> + * Starts a tag switch to the provided tag value and tag
> type.  Completion for
> + * the tag switch must be checked for separately.
> + * This function does NOT update the
> + * work queue entry in dram to match tag value and type, so the
> application must
> + * keep track of these if they are important to the application.
> + * This tag switch command must not be used for switches to NULL, as
> the tag
> + * switch pending bit will be set by the switch request, but never
> cleared by
> + * the hardware.
> + *
> + * This function must be used for tag switches from NULL.
> + *
> + * This function does no checks, so the caller must ensure that any
> previous tag
> + * switch has completed.
> + *
> + * @param wqp      pointer to work queue entry to submit.  This
> entry is
> + *                 updated to match the other parameters
> + * @param tag      tag value to be assigned to work queue entry
> + * @param tag_type type of tag
> + * @param group    group value for the work queue entry.
> + */
> +static inline void cvmx_pow_tag_sw_full_nocheck(cvmx_wqe_t *wqp, u32
> tag,
> +						cvmx_pow_tag_type_t
> tag_type, u64 group)
> +{
> +	union cvmx_pow_tag_req_addr ptr;
> +	cvmx_pow_tag_req_t tag_req;
> +	unsigned int node = cvmx_get_node_num();
> +	u64 wqp_phys = cvmx_ptr_to_phys(wqp);
> +
> +	if (CVMX_ENABLE_POW_CHECKS) {
> +		cvmx_pow_tag_info_t current_tag;
> +
> +		__cvmx_pow_warn_if_pending_switch(__func__);
> +		current_tag = cvmx_pow_get_current_tag();
> +		cvmx_warn_if(current_tag.tag_type ==
> CVMX_POW_TAG_TYPE_NULL_NULL,
> +			     "%s called with NULL_NULL tag\n",
> __func__);
> +		cvmx_warn_if((current_tag.tag_type == tag_type) &&
> (current_tag.tag == tag),
> +			     "%s called to perform a tag switch to the
> same tag\n", __func__);
> +		cvmx_warn_if(
> +			tag_type == CVMX_POW_TAG_TYPE_NULL,
> +			"%s called to perform a tag switch to NULL. Use
> cvmx_pow_tag_sw_null() instead\n",
> +			__func__);
> +		if ((wqp != cvmx_phys_to_ptr(0x80)) &&
> cvmx_pow_get_current_wqp())
> +			cvmx_warn_if(wqp != cvmx_pow_get_current_wqp(),
> +				     "%s passed WQE(%p) doesn't match
> the address in the POW(%p)\n",
> +				     __func__, wqp,
> cvmx_pow_get_current_wqp());
> +	}
> +
> +	/*
> +	 * Note that WQE in DRAM is not updated here, as the POW does
> not
> +	 * read from DRAM once the WQE is in flight.  See hardware
> manual
> +	 * for complete details. It is the application's responsibility
> to
> +	 * keep track of the current tag value if that is important.
> +	 */
> +	tag_req.u64 = 0;
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		unsigned int xgrp;
> +
> +		if (wqp_phys != 0x80) {
> +			/* If WQE is valid, use its XGRP:
> +			 * WQE GRP is 10 bits, and is mapped
> +			 * to legacy GRP + QoS, includes node number.
> +			 */
> +			xgrp = wqp->word1.cn78xx.grp;
> +			/* Use XGRP[node] too */
> +			node = xgrp >> 8;
> +			/* Modify XGRP with legacy group # from arg */
> +			xgrp &= ~0xf8;
> +			xgrp |= 0xf8 & (group << 3);
> +
> +		} else {
> +			/* If no WQE, build XGRP with QoS=0 and current
> node */
> +			xgrp = group << 3;
> +			xgrp |= node << 8;
> +		}
> +		tag_req.s_cn78xx_other.op = CVMX_POW_TAG_OP_SWTAG_FULL;
> +		tag_req.s_cn78xx_other.type = tag_type;
> +		tag_req.s_cn78xx_other.grp = xgrp;
> +		tag_req.s_cn78xx_other.wqp = wqp_phys;
> +	} else if (octeon_has_feature(OCTEON_FEATURE_CN68XX_WQE)) {
> +		tag_req.s_cn68xx_other.op = CVMX_POW_TAG_OP_SWTAG_FULL;
> +		tag_req.s_cn68xx_other.tag = tag;
> +		tag_req.s_cn68xx_other.type = tag_type;
> +		tag_req.s_cn68xx_other.grp = group;
> +	} else {
> +		tag_req.s_cn38xx.op = CVMX_POW_TAG_OP_SWTAG_FULL;
> +		tag_req.s_cn38xx.tag = tag;
> +		tag_req.s_cn38xx.type = tag_type;
> +		tag_req.s_cn38xx.grp = group;
> +	}
> +	ptr.u64 = 0;
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		ptr.s_cn78xx.mem_region = CVMX_IO_SEG;
> +		ptr.s_cn78xx.is_io = 1;
> +		ptr.s_cn78xx.did = CVMX_OCT_DID_TAG_SWTAG;
> +		ptr.s_cn78xx.node = node;
> +		ptr.s_cn78xx.tag = tag;
> +	} else {
> +		ptr.s.mem_region = CVMX_IO_SEG;
> +		ptr.s.is_io = 1;
> +		ptr.s.did = CVMX_OCT_DID_TAG_SWTAG;
> +		ptr.s.addr = wqp_phys;
> +	}
> +	/* Once this store arrives at POW, it will attempt the switch
> +	   software must wait for the switch to complete separately */
> +	cvmx_write_io(ptr.u64, tag_req.u64);
> +}
> +
> +/**
> + * Starts a tag switch to the provided tag value and tag type.
> + * Completion for the tag switch must be checked for separately.
> + * This function does NOT update the work queue entry in dram to
> match tag value
> + * and type, so the application must keep track of these if they are
> important
> + * to the application. This tag switch command must not be used for
> switches
> + * to NULL, as the tag switch pending bit will be set by the switch
> request,
> + * but never cleared by the hardware.
> + *
> + * This function must be used for tag switches from NULL.
> + *
> + * This function waits for any pending tag switches to complete
> + * before requesting the tag switch.
> + *
> + * @param wqp      Pointer to work queue entry to submit.
> + *     This entry is updated to match the other parameters
> + * @param tag      Tag value to be assigned to work queue entry
> + * @param tag_type Type of tag
> + * @param group    Group value for the work queue entry.
> + */
> +static inline void cvmx_pow_tag_sw_full(cvmx_wqe_t *wqp, u32 tag,
> cvmx_pow_tag_type_t tag_type,
> +					u64 group)
> +{
> +	/*
> +	 * Ensure that there is not a pending tag switch, as a tag
> switch cannot
> +	 * be started if a previous switch is still pending.
> +	 */
> +	cvmx_pow_tag_sw_wait();
> +	cvmx_pow_tag_sw_full_nocheck(wqp, tag, tag_type, group);
> +}
> +
> +/**
> + * Switch to a NULL tag, which ends any ordering or
> + * synchronization provided by the POW for the current
> + * work queue entry.  This operation completes immediately,
> + * so completion should not be waited for.
> + * This function does NOT wait for previous tag switches to
> complete,
> + * so the caller must ensure that any previous tag switches have
> completed.
> + */
> +static inline void cvmx_pow_tag_sw_null_nocheck(void)
> +{
> +	union cvmx_pow_tag_req_addr ptr;
> +	cvmx_pow_tag_req_t tag_req;
> +
> +	if (CVMX_ENABLE_POW_CHECKS) {
> +		cvmx_pow_tag_info_t current_tag;
> +
> +		__cvmx_pow_warn_if_pending_switch(__func__);
> +		current_tag = cvmx_pow_get_current_tag();
> +		cvmx_warn_if(current_tag.tag_type ==
> CVMX_POW_TAG_TYPE_NULL_NULL,
> +			     "%s called with NULL_NULL tag\n",
> __func__);
> +		cvmx_warn_if(current_tag.tag_type ==
> CVMX_POW_TAG_TYPE_NULL,
> +			     "%s called when we already have a NULL
> tag\n", __func__);
> +	}
> +	tag_req.u64 = 0;
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		tag_req.s_cn78xx_other.op = CVMX_POW_TAG_OP_SWTAG;
> +		tag_req.s_cn78xx_other.type = CVMX_POW_TAG_TYPE_NULL;
> +	} else if (octeon_has_feature(OCTEON_FEATURE_CN68XX_WQE)) {
> +		tag_req.s_cn68xx_other.op = CVMX_POW_TAG_OP_SWTAG;
> +		tag_req.s_cn68xx_other.type = CVMX_POW_TAG_TYPE_NULL;
> +	} else {
> +		tag_req.s_cn38xx.op = CVMX_POW_TAG_OP_SWTAG;
> +		tag_req.s_cn38xx.type = CVMX_POW_TAG_TYPE_NULL;
> +	}
> +	ptr.u64 = 0;
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		ptr.s_cn78xx.mem_region = CVMX_IO_SEG;
> +		ptr.s_cn78xx.is_io = 1;
> +		ptr.s_cn78xx.did = CVMX_OCT_DID_TAG_TAG1;
> +		ptr.s_cn78xx.node = cvmx_get_node_num();
> +	} else {
> +		ptr.s.mem_region = CVMX_IO_SEG;
> +		ptr.s.is_io = 1;
> +		ptr.s.did = CVMX_OCT_DID_TAG_TAG1;
> +	}
> +	cvmx_write_io(ptr.u64, tag_req.u64);
> +}
> +
> +/**
> + * Switch to a NULL tag, which ends any ordering or
> + * synchronization provided by the POW for the current
> + * work queue entry.  This operation completes immediately,
> + * so completion should not be waited for.
> + * This function waits for any pending tag switches to complete
> + * before requesting the switch to NULL.
> + */
> +static inline void cvmx_pow_tag_sw_null(void)
> +{
> +	/*
> +	 * Ensure that there is not a pending tag switch, as a tag
> switch cannot
> +	 * be started if a previous switch is still pending.
> +	 */
> +	cvmx_pow_tag_sw_wait();
> +	cvmx_pow_tag_sw_null_nocheck();
> +}
> +
> +/**
> + * Submits work to an input queue.
> + * This function updates the work queue entry in DRAM to match the
> arguments given.
> + * Note that the tag provided is for the work queue entry submitted,
> and
> + * is unrelated to the tag that the core currently holds.
> + *
> + * @param wqp      pointer to work queue entry to submit.
> + *                 This entry is updated to match the other
> parameters
> + * @param tag      tag value to be assigned to work queue entry
> + * @param tag_type type of tag
> + * @param qos      Input queue to add to.
> + * @param grp      group value for the work queue entry.
> + */
> +static inline void cvmx_pow_work_submit(cvmx_wqe_t *wqp, u32 tag,
> cvmx_pow_tag_type_t tag_type,
> +					u64 qos, u64 grp)
> +{
> +	union cvmx_pow_tag_req_addr ptr;
> +	cvmx_pow_tag_req_t tag_req;
> +
> +	tag_req.u64 = 0;
> +	ptr.u64 = 0;
> +
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		unsigned int node = cvmx_get_node_num();
> +		unsigned int xgrp;
> +
> +		xgrp = (grp & 0x1f) << 3;
> +		xgrp |= (qos & 7);
> +		xgrp |= 0x300 & (node << 8);
> +
> +		wqp->word1.cn78xx.rsvd_0 = 0;
> +		wqp->word1.cn78xx.rsvd_1 = 0;
> +		wqp->word1.cn78xx.tag = tag;
> +		wqp->word1.cn78xx.tag_type = tag_type;
> +		wqp->word1.cn78xx.grp = xgrp;
> +		CVMX_SYNCWS;
> +
> +		tag_req.s_cn78xx_other.op = CVMX_POW_TAG_OP_ADDWQ;
> +		tag_req.s_cn78xx_other.type = tag_type;
> +		tag_req.s_cn78xx_other.wqp = cvmx_ptr_to_phys(wqp);
> +		tag_req.s_cn78xx_other.grp = xgrp;
> +
> +		ptr.s_cn78xx.did = 0x66; // CVMX_OCT_DID_TAG_TAG6;
> +		ptr.s_cn78xx.mem_region = CVMX_IO_SEG;
> +		ptr.s_cn78xx.is_io = 1;
> +		ptr.s_cn78xx.node = node;
> +		ptr.s_cn78xx.tag = tag;
> +	} else if (octeon_has_feature(OCTEON_FEATURE_CN68XX_WQE)) {
> +		/* Reset all reserved bits */
> +		wqp->word1.cn68xx.zero_0 = 0;
> +		wqp->word1.cn68xx.zero_1 = 0;
> +		wqp->word1.cn68xx.zero_2 = 0;
> +		wqp->word1.cn68xx.qos = qos;
> +		wqp->word1.cn68xx.grp = grp;
> +
> +		wqp->word1.tag = tag;
> +		wqp->word1.tag_type = tag_type;
> +
> +		tag_req.s_cn68xx_add.op = CVMX_POW_TAG_OP_ADDWQ;
> +		tag_req.s_cn68xx_add.type = tag_type;
> +		tag_req.s_cn68xx_add.tag = tag;
> +		tag_req.s_cn68xx_add.qos = qos;
> +		tag_req.s_cn68xx_add.grp = grp;
> +
> +		ptr.s.mem_region = CVMX_IO_SEG;
> +		ptr.s.is_io = 1;
> +		ptr.s.did = CVMX_OCT_DID_TAG_TAG1;
> +		ptr.s.addr = cvmx_ptr_to_phys(wqp);
> +	} else {
> +		/* Reset all reserved bits */
> +		wqp->word1.cn38xx.zero_2 = 0;
> +		wqp->word1.cn38xx.qos = qos;
> +		wqp->word1.cn38xx.grp = grp;
> +
> +		wqp->word1.tag = tag;
> +		wqp->word1.tag_type = tag_type;
> +
> +		tag_req.s_cn38xx.op = CVMX_POW_TAG_OP_ADDWQ;
> +		tag_req.s_cn38xx.type = tag_type;
> +		tag_req.s_cn38xx.tag = tag;
> +		tag_req.s_cn38xx.qos = qos;
> +		tag_req.s_cn38xx.grp = grp;
> +
> +		ptr.s.mem_region = CVMX_IO_SEG;
> +		ptr.s.is_io = 1;
> +		ptr.s.did = CVMX_OCT_DID_TAG_TAG1;
> +		ptr.s.addr = cvmx_ptr_to_phys(wqp);
> +	}
> +	/* SYNC write to memory before the work submit.
> +	 * This is necessary as POW may read values from DRAM at this
> time */
> +	CVMX_SYNCWS;
> +	cvmx_write_io(ptr.u64, tag_req.u64);
> +}
> +
> +/**
> + * This function sets the group mask for a core.  The group mask
> + * indicates which groups each core will accept work from. There are
> + * 16 groups.
> + *
> + * @param core_num   core to apply mask to
> + * @param mask   Group mask, one bit for up to 64 groups.
> + *               Each 1 bit in the mask enables the core to accept
> work from
> + *               the corresponding group.
> + *               The CN68XX supports 64 groups, earlier models only
> support
> + *               16 groups.
> + *
> + * The CN78XX in backwards compatibility mode allows up to 32
> groups,
> + * so the 'mask' argument has one bit for every of the legacy
> + * groups, and a '1' in the mask causes a total of 8 groups
> + * which share the legacy group numbher and 8 qos levels,
> + * to be enabled for the calling processor core.
> + * A '0' in the mask will disable the current core
> + * from receiving work from the associated group.
> + */
> +static inline void cvmx_pow_set_group_mask(u64 core_num, u64 mask)
> +{
> +	u64 valid_mask;
> +	int num_groups = cvmx_pow_num_groups();
> +
> +	if (num_groups >= 64)
> +		valid_mask = ~0ull;
> +	else
> +		valid_mask = (1ull << num_groups) - 1;
> +
> +	if ((mask & valid_mask) == 0) {
> +		printf("ERROR: %s empty group mask disables work on
> core# %llu, ignored.\n",
> +		       __func__, (unsigned long long)core_num);
> +		return;
> +	}
> +	cvmx_warn_if(mask & (~valid_mask), "%s group number range
> exceeded: %#llx\n", __func__,
> +		     (unsigned long long)mask);
> +
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		unsigned int mask_set;
> +		cvmx_sso_ppx_sx_grpmskx_t grp_msk;
> +		unsigned int core, node;
> +		unsigned int rix;  /* Register index */
> +		unsigned int grp;  /* Legacy group # */
> +		unsigned int bit;  /* bit index */
> +		unsigned int xgrp; /* native group # */
> +
> +		node = cvmx_coremask_core_to_node(core_num);
> +		core = cvmx_coremask_core_on_node(core_num);
> +
> +		/* 78xx: 256 groups divided into 4 X 64 bit registers
> */
> +		/* 73xx: 64 groups are in one register */
> +		for (rix = 0; rix < (cvmx_sso_num_xgrp() >> 6); rix++)
> {
> +			grp_msk.u64 = 0;
> +			for (bit = 0; bit < 64; bit++) {
> +				/* 8-bit native XGRP number */
> +				xgrp = (rix << 6) | bit;
> +				/* Legacy 5-bit group number */
> +				grp = (xgrp >> 3) & 0x1f;
> +				/* Inspect legacy mask by legacy group
> */
> +				if (mask & (1ull << grp))
> +					grp_msk.s.grp_msk |= 1ull <<
> bit;
> +				/* Pre-set to all 0's */
> +			}
> +			for (mask_set = 0; mask_set <
> cvmx_sso_num_maskset(); mask_set++) {
> +				csr_wr_node(node,
> CVMX_SSO_PPX_SX_GRPMSKX(core, mask_set, rix),
> +					    grp_msk.u64);
> +			}
> +		}
> +	} else if (octeon_has_feature(OCTEON_FEATURE_CN68XX_WQE)) {
> +		cvmx_sso_ppx_grp_msk_t grp_msk;
> +
> +		grp_msk.s.grp_msk = mask;
> +		csr_wr(CVMX_SSO_PPX_GRP_MSK(core_num), grp_msk.u64);
> +	} else {
> +		cvmx_pow_pp_grp_mskx_t grp_msk;
> +
> +		grp_msk.u64 = csr_rd(CVMX_POW_PP_GRP_MSKX(core_num));
> +		grp_msk.s.grp_msk = mask & 0xffff;
> +		csr_wr(CVMX_POW_PP_GRP_MSKX(core_num), grp_msk.u64);
> +	}
> +}
> +
> +/**
> + * This function gets the group mask for a core.  The group mask
> + * indicates which groups each core will accept work from.
> + *
> + * @param core_num   core to apply mask to
> + * @return	Group mask, one bit for up to 64 groups.
> + *               Each 1 bit in the mask enables the core to accept
> work from
> + *               the corresponding group.
> + *               The CN68XX supports 64 groups, earlier models only
> support
> + *               16 groups.
> + *
> + * The CN78XX in backwards compatibility mode allows up to 32
> groups,
> + * so the 'mask' argument has one bit for every of the legacy
> + * groups, and a '1' in the mask causes a total of 8 groups
> + * which share the legacy group numbher and 8 qos levels,
> + * to be enabled for the calling processor core.
> + * A '0' in the mask will disable the current core
> + * from receiving work from the associated group.
> + */
> +static inline u64 cvmx_pow_get_group_mask(u64 core_num)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		cvmx_sso_ppx_sx_grpmskx_t grp_msk;
> +		unsigned int core, node, i;
> +		int rix; /* Register index */
> +		u64 mask = 0;
> +
> +		node = cvmx_coremask_core_to_node(core_num);
> +		core = cvmx_coremask_core_on_node(core_num);
> +
> +		/* 78xx: 256 groups divided into 4 X 64 bit registers
> */
> +		/* 73xx: 64 groups are in one register */
> +		for (rix = (cvmx_sso_num_xgrp() >> 6) - 1; rix >= 0;
> rix--) {
> +			/* read only mask_set=0 (both 'set' was written
> same) */
> +			grp_msk.u64 = csr_rd_node(node,
> CVMX_SSO_PPX_SX_GRPMSKX(core, 0, rix));
> +			/* ASSUME: (this is how mask bits got written)
> */
> +			/* grp_mask[7:0]: all bits 0..7 are same */
> +			/* grp_mask[15:8]: all bits 8..15 are same, etc
> */
> +			/* DO: mask[7:0] =
> grp_mask.u64[56,48,40,32,24,16,8,0] */
> +			for (i = 0; i < 8; i++)
> +				mask |= (grp_msk.u64 & ((u64)1 << (i *
> 8))) >> (7 * i);
> +			/* we collected 8 MSBs in mask[7:0], <<=8 and
> continue */
> +			if (cvmx_likely(rix != 0))
> +				mask <<= 8;
> +		}
> +		return mask & 0xFFFFFFFF;
> +	} else if (octeon_has_feature(OCTEON_FEATURE_CN68XX_WQE)) {
> +		cvmx_sso_ppx_grp_msk_t grp_msk;
> +
> +		grp_msk.u64 = csr_rd(CVMX_SSO_PPX_GRP_MSK(core_num));
> +		return grp_msk.u64;
> +	} else {
> +		cvmx_pow_pp_grp_mskx_t grp_msk;
> +
> +		grp_msk.u64 = csr_rd(CVMX_POW_PP_GRP_MSKX(core_num));
> +		return grp_msk.u64 & 0xffff;
> +	}
> +}
> +
> +/*
> + * Returns 0 if 78xx(73xx,75xx) is not programmed in legacy
> compatible mode
> + * Returns 1 if 78xx(73xx,75xx) is programmed in legacy compatible
> mode
> + * Returns 1 if octeon model is not 78xx(73xx,75xx)
> + */
> +static inline u64 cvmx_pow_is_legacy78mode(u64 core_num)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		cvmx_sso_ppx_sx_grpmskx_t grp_msk0, grp_msk1;
> +		unsigned int core, node, i;
> +		int rix; /* Register index */
> +		u64 mask = 0;
> +
> +		node = cvmx_coremask_core_to_node(core_num);
> +		core = cvmx_coremask_core_on_node(core_num);
> +
> +		/* 78xx: 256 groups divided into 4 X 64 bit registers
> */
> +		/* 73xx: 64 groups are in one register */
> +		/* 1) in order for the 78_SSO to be in legacy
> compatible mode
> +		 * the both mask_sets should be programmed the same */
> +		for (rix = (cvmx_sso_num_xgrp() >> 6) - 1; rix >= 0;
> rix--) {
> +			/* read mask_set=0 (both 'set' was written
> same) */
> +			grp_msk0.u64 = csr_rd_node(node,
> CVMX_SSO_PPX_SX_GRPMSKX(core, 0, rix));
> +			grp_msk1.u64 = csr_rd_node(node,
> CVMX_SSO_PPX_SX_GRPMSKX(core, 1, rix));
> +			if (grp_msk0.u64 != grp_msk1.u64) {
> +				return 0;
> +			}
> +			/* (this is how mask bits should be written) */
> +			/* grp_mask[7:0]: all bits 0..7 are same */
> +			/* grp_mask[15:8]: all bits 8..15 are same, etc
> */
> +			/* 2) in order for the 78_SSO to be in legacy
> compatible
> +			 * mode above should be true (test only
> mask_set=0 */
> +			for (i = 0; i < 8; i++) {
> +				mask = (grp_msk0.u64 >> (i << 3)) &
> 0xFF;
> +				if (!(mask == 0 || mask == 0xFF)) {
> +					return 0;
> +				}
> +			}
> +		}
> +		/* if we come here, the 78_SSO is in legacy compatible
> mode */
> +	}
> +	return 1; /* the SSO/POW is in legacy (or compatible) mode */
> +}
> +
> +/**
> + * This function sets POW static priorities for a core. Each input
> queue has
> + * an associated priority value.
> + *
> + * @param core_num   core to apply priorities to
> + * @param priority   Vector of 8 priorities, one per POW Input Queue
> (0-7).
> + *                   Highest priority is 0 and lowest is 7. A
> priority value
> + *                   of 0xF instructs POW to skip the Input Queue
> when
> + *                   scheduling to this specific core.
> + *                   NOTE: priorities should not have gaps in
> values, meaning
> + *                         {0,1,1,1,1,1,1,1} is a valid
> configuration while
> + *                         {0,2,2,2,2,2,2,2} is not.
> + */
> +static inline void cvmx_pow_set_priority(u64 core_num, const u8
> priority[])
> +{
> +	/* Detect gaps between priorities and flag error */
> +	if (!octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		int i;
> +		u32 prio_mask = 0;
> +
> +		for (i = 0; i < 8; i++)
> +			if (priority[i] != 0xF)
> +				prio_mask |= 1 << priority[i];
> +
> +		if (prio_mask ^ ((1 << cvmx_pop(prio_mask)) - 1)) {
> +			debug("ERROR: POW static priorities should be
> contiguous (0x%llx)\n",
> +			      (unsigned long long)prio_mask);
> +			return;
> +		}
> +	}
> +
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		unsigned int group;
> +		unsigned int node = cvmx_get_node_num();
> +		cvmx_sso_grpx_pri_t grp_pri;
> +
> +		/*grp_pri.s.weight = 0x3f; these will be anyway
> overwritten */
> +		/*grp_pri.s.affinity = 0xf; by the next
> csr_rd_node(..), */
> +
> +		for (group = 0; group < cvmx_sso_num_xgrp(); group++) {
> +			grp_pri.u64 = csr_rd_node(node,
> CVMX_SSO_GRPX_PRI(group));
> +			grp_pri.s.pri = priority[group & 0x7];
> +			csr_wr_node(node, CVMX_SSO_GRPX_PRI(group),
> grp_pri.u64);
> +		}
> +
> +	} else if (octeon_has_feature(OCTEON_FEATURE_CN68XX_WQE)) {
> +		cvmx_sso_ppx_qos_pri_t qos_pri;
> +
> +		qos_pri.u64 = csr_rd(CVMX_SSO_PPX_QOS_PRI(core_num));
> +		qos_pri.s.qos0_pri = priority[0];
> +		qos_pri.s.qos1_pri = priority[1];
> +		qos_pri.s.qos2_pri = priority[2];
> +		qos_pri.s.qos3_pri = priority[3];
> +		qos_pri.s.qos4_pri = priority[4];
> +		qos_pri.s.qos5_pri = priority[5];
> +		qos_pri.s.qos6_pri = priority[6];
> +		qos_pri.s.qos7_pri = priority[7];
> +		csr_wr(CVMX_SSO_PPX_QOS_PRI(core_num), qos_pri.u64);
> +	} else {
> +		/* POW priorities on CN5xxx .. CN66XX */
> +		cvmx_pow_pp_grp_mskx_t grp_msk;
> +
> +		grp_msk.u64 = csr_rd(CVMX_POW_PP_GRP_MSKX(core_num));
> +		grp_msk.s.qos0_pri = priority[0];
> +		grp_msk.s.qos1_pri = priority[1];
> +		grp_msk.s.qos2_pri = priority[2];
> +		grp_msk.s.qos3_pri = priority[3];
> +		grp_msk.s.qos4_pri = priority[4];
> +		grp_msk.s.qos5_pri = priority[5];
> +		grp_msk.s.qos6_pri = priority[6];
> +		grp_msk.s.qos7_pri = priority[7];
> +
> +		csr_wr(CVMX_POW_PP_GRP_MSKX(core_num), grp_msk.u64);
> +	}
> +}
> +
> +/**
> + * This function gets POW static priorities for a core. Each input
> queue has
> + * an associated priority value.
> + *
> + * @param[in]  core_num core to get priorities for
> + * @param[out] priority Pointer to u8[] where to return priorities
> + *			Vector of 8 priorities, one per POW Input Queue
> (0-7).
> + *			Highest priority is 0 and lowest is 7. A
> priority value
> + *			of 0xF instructs POW to skip the Input Queue
> when
> + *			scheduling to this specific core.
> + *                   NOTE: priorities should not have gaps in
> values, meaning
> + *                         {0,1,1,1,1,1,1,1} is a valid
> configuration while
> + *                         {0,2,2,2,2,2,2,2} is not.
> + */
> +static inline void cvmx_pow_get_priority(u64 core_num, u8
> priority[])
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		unsigned int group;
> +		unsigned int node = cvmx_get_node_num();
> +		cvmx_sso_grpx_pri_t grp_pri;
> +
> +		/* read priority only from the first 8 groups */
> +		/* the next groups are programmed the same
> (periodicaly) */
> +		for (group = 0; group < 8 /*cvmx_sso_num_xgrp() */;
> group++) {
> +			grp_pri.u64 = csr_rd_node(node,
> CVMX_SSO_GRPX_PRI(group));
> +			priority[group /* & 0x7 */] = grp_pri.s.pri;
> +		}
> +
> +	} else if (octeon_has_feature(OCTEON_FEATURE_CN68XX_WQE)) {
> +		cvmx_sso_ppx_qos_pri_t qos_pri;
> +
> +		qos_pri.u64 = csr_rd(CVMX_SSO_PPX_QOS_PRI(core_num));
> +		priority[0] = qos_pri.s.qos0_pri;
> +		priority[1] = qos_pri.s.qos1_pri;
> +		priority[2] = qos_pri.s.qos2_pri;
> +		priority[3] = qos_pri.s.qos3_pri;
> +		priority[4] = qos_pri.s.qos4_pri;
> +		priority[5] = qos_pri.s.qos5_pri;
> +		priority[6] = qos_pri.s.qos6_pri;
> +		priority[7] = qos_pri.s.qos7_pri;
> +	} else {
> +		/* POW priorities on CN5xxx .. CN66XX */
> +		cvmx_pow_pp_grp_mskx_t grp_msk;
> +
> +		grp_msk.u64 = csr_rd(CVMX_POW_PP_GRP_MSKX(core_num));
> +		priority[0] = grp_msk.s.qos0_pri;
> +		priority[1] = grp_msk.s.qos1_pri;
> +		priority[2] = grp_msk.s.qos2_pri;
> +		priority[3] = grp_msk.s.qos3_pri;
> +		priority[4] = grp_msk.s.qos4_pri;
> +		priority[5] = grp_msk.s.qos5_pri;
> +		priority[6] = grp_msk.s.qos6_pri;
> +		priority[7] = grp_msk.s.qos7_pri;
> +	}
> +
> +	/* Detect gaps between priorities and flag error - (optional)
> */
> +	if (!octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		int i;
> +		u32 prio_mask = 0;
> +
> +		for (i = 0; i < 8; i++)
> +			if (priority[i] != 0xF)
> +				prio_mask |= 1 << priority[i];
> +
> +		if (prio_mask ^ ((1 << cvmx_pop(prio_mask)) - 1)) {
> +			debug("ERROR:%s: POW static priorities should
> be contiguous (0x%llx)\n",
> +			      __func__, (unsigned long long)prio_mask);
> +			return;
> +		}
> +	}
> +}
> +
> +static inline void cvmx_sso_get_group_priority(int node, cvmx_xgrp_t
> xgrp, int *priority,
> +					       int *weight, int
> *affinity)
> +{
> +	cvmx_sso_grpx_pri_t grp_pri;
> +
> +	if (!octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		debug("ERROR: %s is not supported on this chip)\n",
> __func__);
> +		return;
> +	}
> +
> +	grp_pri.u64 = csr_rd_node(node, CVMX_SSO_GRPX_PRI(xgrp.xgrp));
> +	*affinity = grp_pri.s.affinity;
> +	*priority = grp_pri.s.pri;
> +	*weight = grp_pri.s.weight;
> +}
> +
> +/**
> + * Performs a tag switch and then an immediate deschedule. This
> completes
> + * immediately, so completion must not be waited for.  This function
> does NOT
> + * update the wqe in DRAM to match arguments.
> + *
> + * This function does NOT wait for any prior tag switches to
> complete, so the
> + * calling code must do this.
> + *
> + * Note the following CAVEAT of the Octeon HW behavior when
> + * re-scheduling DE-SCHEDULEd items whose (next) state is
> + * ORDERED:
> + *   - If there are no switches pending at the time that the
> + *     HW executes the de-schedule, the HW will only re-schedule
> + *     the head of the FIFO associated with the given tag. This
> + *     means that in many respects, the HW treats this ORDERED
> + *     tag as an ATOMIC tag. Note that in the SWTAG_DESCH
> + *     case (to an ORDERED tag), the HW will do the switch
> + *     before the deschedule whenever it is possible to do
> + *     the switch immediately, so it may often look like
> + *     this case.
> + *   - If there is a pending switch to ORDERED at the time
> + *     the HW executes the de-schedule, the HW will perform
> + *     the switch at the time it re-schedules, and will be
> + *     able to reschedule any/all of the entries with the
> + *     same tag.
> + * Due to this behavior, the RECOMMENDATION to software is
> + * that they have a (next) state of ATOMIC when they
> + * DE-SCHEDULE. If an ORDERED tag is what was really desired,
> + * SW can choose to immediately switch to an ORDERED tag
> + * after the work (that has an ATOMIC tag) is re-scheduled.
> + * Note that since there are never any tag switches pending
> + * when the HW re-schedules, this switch can be IMMEDIATE upon
> + * the reception of the pointer during the re-schedule.
> + *
> + * @param tag      New tag value
> + * @param tag_type New tag type
> + * @param group    New group value
> + * @param no_sched Control whether this work queue entry will be
> rescheduled.
> + *                 - 1 : don't schedule this work
> + *                 - 0 : allow this work to be scheduled.
> + */
> +static inline void cvmx_pow_tag_sw_desched_nocheck(u32 tag,
> cvmx_pow_tag_type_t tag_type, u64 group,
> +						   u64 no_sched)
> +{
> +	union cvmx_pow_tag_req_addr ptr;
> +	cvmx_pow_tag_req_t tag_req;
> +
> +	if (CVMX_ENABLE_POW_CHECKS) {
> +		cvmx_pow_tag_info_t current_tag;
> +
> +		__cvmx_pow_warn_if_pending_switch(__func__);
> +		current_tag = cvmx_pow_get_current_tag();
> +		cvmx_warn_if(current_tag.tag_type ==
> CVMX_POW_TAG_TYPE_NULL_NULL,
> +			     "%s called with NULL_NULL tag\n",
> __func__);
> +		cvmx_warn_if(current_tag.tag_type ==
> CVMX_POW_TAG_TYPE_NULL,
> +			     "%s called with NULL tag. Deschedule not
> allowed from NULL state\n",
> +			     __func__);
> +		cvmx_warn_if((current_tag.tag_type !=
> CVMX_POW_TAG_TYPE_ATOMIC) &&
> +			     (tag_type != CVMX_POW_TAG_TYPE_ATOMIC),
> +			     "%s called where neither the before or
> after tag is ATOMIC\n",
> +			     __func__);
> +	}
> +	tag_req.u64 = 0;
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		cvmx_wqe_t *wqp = cvmx_pow_get_current_wqp();
> +
> +		if (!wqp) {
> +			debug("ERROR: Failed to get WQE, %s\n",
> __func__);
> +			return;
> +		}
> +		group &= 0x1f;
> +		wqp->word1.cn78xx.tag = tag;
> +		wqp->word1.cn78xx.tag_type = tag_type;
> +		wqp->word1.cn78xx.grp = group << 3;
> +		CVMX_SYNCWS;
> +		tag_req.s_cn78xx_other.op =
> CVMX_POW_TAG_OP_SWTAG_DESCH;
> +		tag_req.s_cn78xx_other.type = tag_type;
> +		tag_req.s_cn78xx_other.grp = group << 3;
> +		tag_req.s_cn78xx_other.no_sched = no_sched;
> +	} else if (octeon_has_feature(OCTEON_FEATURE_CN68XX_WQE)) {
> +		group &= 0x3f;
> +		tag_req.s_cn68xx_other.op =
> CVMX_POW_TAG_OP_SWTAG_DESCH;
> +		tag_req.s_cn68xx_other.tag = tag;
> +		tag_req.s_cn68xx_other.type = tag_type;
> +		tag_req.s_cn68xx_other.grp = group;
> +		tag_req.s_cn68xx_other.no_sched = no_sched;
> +	} else {
> +		group &= 0x0f;
> +		tag_req.s_cn38xx.op = CVMX_POW_TAG_OP_SWTAG_DESCH;
> +		tag_req.s_cn38xx.tag = tag;
> +		tag_req.s_cn38xx.type = tag_type;
> +		tag_req.s_cn38xx.grp = group;
> +		tag_req.s_cn38xx.no_sched = no_sched;
> +	}
> +	ptr.u64 = 0;
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		ptr.s.mem_region = CVMX_IO_SEG;
> +		ptr.s.is_io = 1;
> +		ptr.s.did = CVMX_OCT_DID_TAG_TAG3;
> +		ptr.s_cn78xx.node = cvmx_get_node_num();
> +		ptr.s_cn78xx.tag = tag;
> +	} else {
> +		ptr.s.mem_region = CVMX_IO_SEG;
> +		ptr.s.is_io = 1;
> +		ptr.s.did = CVMX_OCT_DID_TAG_TAG3;
> +	}
> +	cvmx_write_io(ptr.u64, tag_req.u64);
> +}
> +
> +/**
> + * Performs a tag switch and then an immediate deschedule. This
> completes
> + * immediately, so completion must not be waited for.  This function
> does NOT
> + * update the wqe in DRAM to match arguments.
> + *
> + * This function waits for any prior tag switches to complete, so
> the
> + * calling code may call this function with a pending tag switch.
> + *
> + * Note the following CAVEAT of the Octeon HW behavior when
> + * re-scheduling DE-SCHEDULEd items whose (next) state is
> + * ORDERED:
> + *   - If there are no switches pending at the time that the
> + *     HW executes the de-schedule, the HW will only re-schedule
> + *     the head of the FIFO associated with the given tag. This
> + *     means that in many respects, the HW treats this ORDERED
> + *     tag as an ATOMIC tag. Note that in the SWTAG_DESCH
> + *     case (to an ORDERED tag), the HW will do the switch
> + *     before the deschedule whenever it is possible to do
> + *     the switch immediately, so it may often look like
> + *     this case.
> + *   - If there is a pending switch to ORDERED at the time
> + *     the HW executes the de-schedule, the HW will perform
> + *     the switch at the time it re-schedules, and will be
> + *     able to reschedule any/all of the entries with the
> + *     same tag.
> + * Due to this behavior, the RECOMMENDATION to software is
> + * that they have a (next) state of ATOMIC when they
> + * DE-SCHEDULE. If an ORDERED tag is what was really desired,
> + * SW can choose to immediately switch to an ORDERED tag
> + * after the work (that has an ATOMIC tag) is re-scheduled.
> + * Note that since there are never any tag switches pending
> + * when the HW re-schedules, this switch can be IMMEDIATE upon
> + * the reception of the pointer during the re-schedule.
> + *
> + * @param tag      New tag value
> + * @param tag_type New tag type
> + * @param group    New group value
> + * @param no_sched Control whether this work queue entry will be
> rescheduled.
> + *                 - 1 : don't schedule this work
> + *                 - 0 : allow this work to be scheduled.
> + */
> +static inline void cvmx_pow_tag_sw_desched(u32 tag,
> cvmx_pow_tag_type_t tag_type, u64 group,
> +					   u64 no_sched)
> +{
> +	/* Need to make sure any writes to the work queue entry are
> complete */
> +	CVMX_SYNCWS;
> +	/* Ensure that there is not a pending tag switch, as a tag
> switch cannot be started
> +	 * if a previous switch is still pending.  */
> +	cvmx_pow_tag_sw_wait();
> +	cvmx_pow_tag_sw_desched_nocheck(tag, tag_type, group,
> no_sched);
> +}
> +
> +/**
> + * Descchedules the current work queue entry.
> + *
> + * @param no_sched no schedule flag value to be set on the work
> queue entry.
> + *     If this is set the entry will not be rescheduled.
> + */
> +static inline void cvmx_pow_desched(u64 no_sched)
> +{
> +	union cvmx_pow_tag_req_addr ptr;
> +	cvmx_pow_tag_req_t tag_req;
> +
> +	if (CVMX_ENABLE_POW_CHECKS) {
> +		cvmx_pow_tag_info_t current_tag;
> +
> +		__cvmx_pow_warn_if_pending_switch(__func__);
> +		current_tag = cvmx_pow_get_current_tag();
> +		cvmx_warn_if(current_tag.tag_type ==
> CVMX_POW_TAG_TYPE_NULL_NULL,
> +			     "%s called with NULL_NULL tag\n",
> __func__);
> +		cvmx_warn_if(current_tag.tag_type ==
> CVMX_POW_TAG_TYPE_NULL,
> +			     "%s called with NULL tag. Deschedule not
> expected from NULL state\n",
> +			     __func__);
> +	}
> +	/* Need to make sure any writes to the work queue entry are
> complete */
> +	CVMX_SYNCWS;
> +
> +	tag_req.u64 = 0;
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		tag_req.s_cn78xx_other.op = CVMX_POW_TAG_OP_DESCH;
> +		tag_req.s_cn78xx_other.no_sched = no_sched;
> +	} else if (octeon_has_feature(OCTEON_FEATURE_CN68XX_WQE)) {
> +		tag_req.s_cn68xx_other.op = CVMX_POW_TAG_OP_DESCH;
> +		tag_req.s_cn68xx_other.no_sched = no_sched;
> +	} else {
> +		tag_req.s_cn38xx.op = CVMX_POW_TAG_OP_DESCH;
> +		tag_req.s_cn38xx.no_sched = no_sched;
> +	}
> +	ptr.u64 = 0;
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		ptr.s_cn78xx.mem_region = CVMX_IO_SEG;
> +		ptr.s_cn78xx.is_io = 1;
> +		ptr.s_cn78xx.did = CVMX_OCT_DID_TAG_TAG3;
> +		ptr.s_cn78xx.node = cvmx_get_node_num();
> +	} else {
> +		ptr.s.mem_region = CVMX_IO_SEG;
> +		ptr.s.is_io = 1;
> +		ptr.s.did = CVMX_OCT_DID_TAG_TAG3;
> +	}
> +	cvmx_write_io(ptr.u64, tag_req.u64);
> +}
> +
> +/*******************************************************************
> ***********/
> +/* OCTEON3-specific
> functions.                                                */
> +/*******************************************************************
> ***********/
> +/**
> + * This function sets the the affinity of group to the cores in
> 78xx.
> + * It sets up all the cores in core_mask to accept work from the
> specified group.
> + *
> + * @param xgrp	Group to accept work from, 0 - 255.
> + * @param core_mask	Mask of all the cores which will accept work
> from this group
> + * @param mask_set	Every core has set of 2 masks which can be set
> to accept work
> + *     from 256 groups. At the time of get_work, cores can choose
> which mask_set
> + *     to get work from. 'mask_set' values range from 0 to 3, where	
> each of the
> + *     two bits represents a mask set. Cores will be added to the
> mask set with
> + *     corresponding bit set, and removed from the mask set with
> corresponding
> + *     bit clear.
> + * Note: cores can only accept work from SSO groups on the same
> node,
> + * so the node number for the group is derived from the core number.
> + */
> +static inline void cvmx_sso_set_group_core_affinity(cvmx_xgrp_t
> xgrp,
> +						    const struct
> cvmx_coremask *core_mask,
> +						    u8 mask_set)
> +{
> +	cvmx_sso_ppx_sx_grpmskx_t grp_msk;
> +	int core;
> +	int grp_index = xgrp.xgrp >> 6;
> +	int bit_pos = xgrp.xgrp % 64;
> +
> +	if (!octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		debug("ERROR: %s is not supported on this chip)\n",
> __func__);
> +		return;
> +	}
> +	cvmx_coremask_for_each_core(core, core_mask)
> +	{
> +		unsigned int node, ncore;
> +		u64 reg_addr;
> +
> +		node = cvmx_coremask_core_to_node(core);
> +		ncore = cvmx_coremask_core_on_node(core);
> +
> +		reg_addr = CVMX_SSO_PPX_SX_GRPMSKX(ncore, 0,
> grp_index);
> +		grp_msk.u64 = csr_rd_node(node, reg_addr);
> +
> +		if (mask_set & 1)
> +			grp_msk.s.grp_msk |= (1ull << bit_pos);
> +		else
> +			grp_msk.s.grp_msk &= ~(1ull << bit_pos);
> +
> +		csr_wr_node(node, reg_addr, grp_msk.u64);
> +
> +		reg_addr = CVMX_SSO_PPX_SX_GRPMSKX(ncore, 1,
> grp_index);
> +		grp_msk.u64 = csr_rd_node(node, reg_addr);
> +
> +		if (mask_set & 2)
> +			grp_msk.s.grp_msk |= (1ull << bit_pos);
> +		else
> +			grp_msk.s.grp_msk &= ~(1ull << bit_pos);
> +
> +		csr_wr_node(node, reg_addr, grp_msk.u64);
> +	}
> +}
> +
> +/**
> + * This function sets the priority and group affinity arbitration
> for each group.
> + *
> + * @param node		Node number
> + * @param xgrp	Group 0 - 255 to apply mask parameters to
> + * @param priority	Priority of the group relative to other groups
> + *     0x0 - highest priority
> + *     0x7 - lowest priority
> + * @param weight	Cross-group arbitration weight to apply to this
> group.
> + *     valid values are 1-63
> + *     h/w default is 0x3f
> + * @param affinity	Processor affinity arbitration weight to apply
> to this group.
> + *     If zero, affinity is disabled.
> + *     valid values are 0-15
> + *     h/w default which is 0xf.
> + * @param modify_mask   mask of the parameters which needs to be
> modified.
> + *     enum cvmx_sso_group_modify_mask
> + *     to modify only priority -- set bit0
> + *     to modify only weight   -- set bit1
> + *     to modify only affinity -- set bit2
> + */
> +static inline void cvmx_sso_set_group_priority(int node, cvmx_xgrp_t
> xgrp, int priority, int weight,
> +					       int affinity,
> +					       enum
> cvmx_sso_group_modify_mask modify_mask)
> +{
> +	cvmx_sso_grpx_pri_t grp_pri;
> +
> +	if (!octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		debug("ERROR: %s is not supported on this chip)\n",
> __func__);
> +		return;
> +	}
> +	if (weight <= 0)
> +		weight = 0x3f; /* Force HW default when out of range */
> +
> +	grp_pri.u64 = csr_rd_node(node, CVMX_SSO_GRPX_PRI(xgrp.xgrp));
> +	if (grp_pri.s.weight == 0)
> +		grp_pri.s.weight = 0x3f;
> +	if (modify_mask & CVMX_SSO_MODIFY_GROUP_PRIORITY)
> +		grp_pri.s.pri = priority;
> +	if (modify_mask & CVMX_SSO_MODIFY_GROUP_WEIGHT)
> +		grp_pri.s.weight = weight;
> +	if (modify_mask & CVMX_SSO_MODIFY_GROUP_AFFINITY)
> +		grp_pri.s.affinity = affinity;
> +	csr_wr_node(node, CVMX_SSO_GRPX_PRI(xgrp.xgrp), grp_pri.u64);
> +}
> +
> +/**
> + * Asynchronous work request.
> + * Only works on CN78XX style SSO.
> + *
> + * Work is requested from the SSO unit, and should later be checked
> with
> + * function cvmx_pow_work_response_async.
> + * This function does NOT wait for previous tag switches to
> complete,
> + * so the caller must ensure that there is not a pending tag switch.
> + *
> + * @param scr_addr Scratch memory address that response will be
> returned to,
> + *     which is either a valid WQE, or a response with the invalid
> bit set.
> + *     Byte address, must be 8 byte aligned.
> + * @param xgrp  Group to receive work for (0-255).
> + * @param wait
> + *     1 to cause response to wait for work to become available (or
> timeout)
> + *     0 to cause response to return immediately
> + */
> +static inline void cvmx_sso_work_request_grp_async_nocheck(int
> scr_addr, cvmx_xgrp_t xgrp,
> +							   cvmx_pow_wai
> t_t wait)
> +{
> +	cvmx_pow_iobdma_store_t data;
> +	unsigned int node = cvmx_get_node_num();
> +
> +	if (CVMX_ENABLE_POW_CHECKS) {
> +		__cvmx_pow_warn_if_pending_switch(__func__);
> +		cvmx_warn_if(!octeon_has_feature(OCTEON_FEATURE_CN78XX_
> WQE), "Not CN78XX");
> +	}
> +	/* scr_addr must be 8 byte aligned */
> +	data.u64 = 0;
> +	data.s_cn78xx.scraddr = scr_addr >> 3;
> +	data.s_cn78xx.len = 1;
> +	data.s_cn78xx.did = CVMX_OCT_DID_TAG_SWTAG;
> +	data.s_cn78xx.grouped = 1;
> +	data.s_cn78xx.index_grp_mask = (node << 8) | xgrp.xgrp;
> +	data.s_cn78xx.wait = wait;
> +	data.s_cn78xx.node = node;
> +
> +	cvmx_send_single(data.u64);
> +}
> +
> +/**
> + * Synchronous work request from the node-local SSO without
> verifying
> + * pending tag switch. It requests work from a specific SSO group.
> + *
> + * @param lgrp The local group number (within the SSO of the node of
> the caller)
> + *     from which to get the work.
> + * @param wait When set, call stalls until work becomes available,
> or times out.
> + *     If not set, returns immediately.
> + *
> + * @return Returns the WQE pointer from SSO.
> + *     Returns NULL if no work was available.
> + */
> +static inline void *cvmx_sso_work_request_grp_sync_nocheck(unsigned
> int lgrp, cvmx_pow_wait_t wait)
> +{
> +	cvmx_pow_load_addr_t ptr;
> +	cvmx_pow_tag_load_resp_t result;
> +	unsigned int node = cvmx_get_node_num() & 3;
> +
> +	if (CVMX_ENABLE_POW_CHECKS) {
> +		__cvmx_pow_warn_if_pending_switch(__func__);
> +		cvmx_warn_if(!octeon_has_feature(OCTEON_FEATURE_CN78XX_
> WQE), "Not CN78XX");
> +	}
> +	ptr.u64 = 0;
> +	ptr.swork_78xx.mem_region = CVMX_IO_SEG;
> +	ptr.swork_78xx.is_io = 1;
> +	ptr.swork_78xx.did = CVMX_OCT_DID_TAG_SWTAG;
> +	ptr.swork_78xx.node = node;
> +	ptr.swork_78xx.grouped = 1;
> +	ptr.swork_78xx.index = (lgrp & 0xff) | node << 8;
> +	ptr.swork_78xx.wait = wait;
> +
> +	result.u64 = csr_rd(ptr.u64);
> +	if (result.s_work.no_work)
> +		return NULL;
> +	else
> +		return cvmx_phys_to_ptr(result.s_work.addr);
> +}
> +
> +/**
> + * Synchronous work request from the node-local SSO.
> + * It requests work from a specific SSO group.
> + * This function waits for any previous tag switch to complete
> before
> + * requesting the new work.
> + *
> + * @param lgrp The node-local group number from which to get the
> work.
> + * @param wait When set, call stalls until work becomes available,
> or times out.
> + *     If not set, returns immediately.
> + *
> + * @return The WQE pointer or NULL, if work is not available.
> + */
> +static inline void *cvmx_sso_work_request_grp_sync(unsigned int
> lgrp, cvmx_pow_wait_t wait)
> +{
> +	cvmx_pow_tag_sw_wait();
> +	return cvmx_sso_work_request_grp_sync_nocheck(lgrp, wait);
> +}
> +
> +/**
> + * This function sets the group mask for a core.  The group mask
> bits
> + * indicate which groups each core will accept work from.
> + *
> + * @param core_num	Processor core to apply mask to.
> + * @param mask_set	7XXX has 2 sets of masks per core.
> + *     Bit 0 represents the first mask set, bit 1 -- the second.
> + * @param xgrp_mask	Group mask array.
> + *     Total number of groups is divided into a number of
> + *     64-bits mask sets. Each bit in the mask, if set, enables
> + *     the core to accept work from the corresponding group.
> + *
> + * NOTE: Each core can be configured to accept work in accordance to
> both
> + * mask sets, with the first having higher precedence over the
> second,
> + * or to accept work in accordance to just one of the two mask sets.
> + * The 'core_num' argument represents a processor core on any node
> + * in a coherent multi-chip system.
> + *
> + * If the 'mask_set' argument is 3, both mask sets are configured
> + * with the same value (which is not typically the intention),
> + * so keep in mind the function needs to be called twice
> + * to set a different value into each of the mask sets,
> + * once with 'mask_set=1' and second time with 'mask_set=2'.
> + */
> +static inline void cvmx_pow_set_xgrp_mask(u64 core_num, u8 mask_set,
> const u64 xgrp_mask[])
> +{
> +	unsigned int grp, node, core;
> +	u64 reg_addr;
> +
> +	if (!octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		debug("ERROR: %s is not supported on this chip)\n",
> __func__);
> +		return;
> +	}
> +
> +	if (CVMX_ENABLE_POW_CHECKS) {
> +		cvmx_warn_if(((mask_set < 1) || (mask_set > 3)),
> "Invalid mask set");
> +	}
> +
> +	if ((mask_set < 1) || (mask_set > 3))
> +		mask_set = 3;
> +
> +	node = cvmx_coremask_core_to_node(core_num);
> +	core = cvmx_coremask_core_on_node(core_num);
> +
> +	for (grp = 0; grp < (cvmx_sso_num_xgrp() >> 6); grp++) {
> +		if (mask_set & 1) {
> +			reg_addr = CVMX_SSO_PPX_SX_GRPMSKX(core, 0,
> grp),
> +			csr_wr_node(node, reg_addr, xgrp_mask[grp]);
> +		}
> +		if (mask_set & 2) {
> +			reg_addr = CVMX_SSO_PPX_SX_GRPMSKX(core, 1,
> grp),
> +			csr_wr_node(node, reg_addr, xgrp_mask[grp]);
> +		}
> +	}
> +}
> +
> +/**
> + * This function gets the group mask for a core.  The group mask
> bits
> + * indicate which groups each core will accept work from.
> + *
> + * @param core_num	Processor core to apply mask to.
> + * @param mask_set	7XXX has 2 sets of masks per core.
> + *     Bit 0 represents the first mask set, bit 1 -- the second.
> + * @param xgrp_mask	Provide pointer to u64 mask[8] output array.
> + *     Total number of groups is divided into a number of
> + *     64-bits mask sets. Each bit in the mask represents
> + *     the core accepts work from the corresponding group.
> + *
> + * NOTE: Each core can be configured to accept work in accordance to
> both
> + * mask sets, with the first having higher precedence over the
> second,
> + * or to accept work in accordance to just one of the two mask sets.
> + * The 'core_num' argument represents a processor core on any node
> + * in a coherent multi-chip system.
> + */
> +static inline void cvmx_pow_get_xgrp_mask(u64 core_num, u8 mask_set,
> u64 *xgrp_mask)
> +{
> +	cvmx_sso_ppx_sx_grpmskx_t grp_msk;
> +	unsigned int grp, node, core;
> +	u64 reg_addr;
> +
> +	if (!octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		debug("ERROR: %s is not supported on this chip)\n",
> __func__);
> +		return;
> +	}
> +
> +	if (CVMX_ENABLE_POW_CHECKS) {
> +		cvmx_warn_if(mask_set != 1 && mask_set != 2, "Invalid
> mask set");
> +	}
> +
> +	node = cvmx_coremask_core_to_node(core_num);
> +	core = cvmx_coremask_core_on_node(core_num);
> +
> +	for (grp = 0; grp < cvmx_sso_num_xgrp() >> 6; grp++) {
> +		if (mask_set & 1) {
> +			reg_addr = CVMX_SSO_PPX_SX_GRPMSKX(core, 0,
> grp),
> +			grp_msk.u64 = csr_rd_node(node, reg_addr);
> +			xgrp_mask[grp] = grp_msk.s.grp_msk;
> +		}
> +		if (mask_set & 2) {
> +			reg_addr = CVMX_SSO_PPX_SX_GRPMSKX(core, 1,
> grp),
> +			grp_msk.u64 = csr_rd_node(node, reg_addr);
> +			xgrp_mask[grp] = grp_msk.s.grp_msk;
> +		}
> +	}
> +}
> +
> +/**
> + * Executes SSO SWTAG command.
> + * This is similar to cvmx_pow_tag_sw() function, but uses linear
> + * (vs. integrated group-qos) group index.
> + */
> +static inline void cvmx_pow_tag_sw_node(cvmx_wqe_t *wqp, u32 tag,
> cvmx_pow_tag_type_t tag_type,
> +					int node)
> +{
> +	union cvmx_pow_tag_req_addr ptr;
> +	cvmx_pow_tag_req_t tag_req;
> +
> +	if
> (cvmx_unlikely(!octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE))) {
> +		debug("ERROR: %s is supported on OCTEON3 only\n",
> __func__);
> +		return;
> +	}
> +	CVMX_SYNCWS;
> +	cvmx_pow_tag_sw_wait();
> +
> +	if (CVMX_ENABLE_POW_CHECKS) {
> +		cvmx_pow_tag_info_t current_tag;
> +
> +		__cvmx_pow_warn_if_pending_switch(__func__);
> +		current_tag = cvmx_pow_get_current_tag();
> +		cvmx_warn_if(current_tag.tag_type ==
> CVMX_POW_TAG_TYPE_NULL_NULL,
> +			     "%s called with NULL_NULL tag\n",
> __func__);
> +		cvmx_warn_if(current_tag.tag_type ==
> CVMX_POW_TAG_TYPE_NULL,
> +			     "%s called with NULL tag\n", __func__);
> +		cvmx_warn_if((current_tag.tag_type == tag_type) &&
> (current_tag.tag == tag),
> +			     "%s called to perform a tag switch to the
> same tag\n", __func__);
> +		cvmx_warn_if(
> +			tag_type == CVMX_POW_TAG_TYPE_NULL,
> +			"%s called to perform a tag switch to NULL. Use
> cvmx_pow_tag_sw_null() instead\n",
> +			__func__);
> +	}
> +	wqp->word1.cn78xx.tag = tag;
> +	wqp->word1.cn78xx.tag_type = tag_type;
> +	CVMX_SYNCWS;
> +
> +	tag_req.u64 = 0;
> +	tag_req.s_cn78xx_other.op = CVMX_POW_TAG_OP_SWTAG;
> +	tag_req.s_cn78xx_other.type = tag_type;
> +
> +	ptr.u64 = 0;
> +	ptr.s_cn78xx.mem_region = CVMX_IO_SEG;
> +	ptr.s_cn78xx.is_io = 1;
> +	ptr.s_cn78xx.did = CVMX_OCT_DID_TAG_SWTAG;
> +	ptr.s_cn78xx.node = node;
> +	ptr.s_cn78xx.tag = tag;
> +	cvmx_write_io(ptr.u64, tag_req.u64);
> +}
> +
> +/**
> + * Executes SSO SWTAG_FULL command.
> + * This is similar to cvmx_pow_tag_sw_full() function, but
> + * uses linear (vs. integrated group-qos) group index.
> + */
> +static inline void cvmx_pow_tag_sw_full_node(cvmx_wqe_t *wqp, u32
> tag, cvmx_pow_tag_type_t tag_type,
> +					     u8 xgrp, int node)
> +{
> +	union cvmx_pow_tag_req_addr ptr;
> +	cvmx_pow_tag_req_t tag_req;
> +	u16 gxgrp;
> +
> +	if
> (cvmx_unlikely(!octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE))) {
> +		debug("ERROR: %s is supported on OCTEON3 only\n",
> __func__);
> +		return;
> +	}
> +	/* Ensure that there is not a pending tag switch, as a tag
> switch cannot be
> +	 * started, if a previous switch is still pending. */
> +	CVMX_SYNCWS;
> +	cvmx_pow_tag_sw_wait();
> +
> +	if (CVMX_ENABLE_POW_CHECKS) {
> +		cvmx_pow_tag_info_t current_tag;
> +
> +		__cvmx_pow_warn_if_pending_switch(__func__);
> +		current_tag = cvmx_pow_get_current_tag();
> +		cvmx_warn_if(current_tag.tag_type ==
> CVMX_POW_TAG_TYPE_NULL_NULL,
> +			     "%s called with NULL_NULL tag\n",
> __func__);
> +		cvmx_warn_if((current_tag.tag_type == tag_type) &&
> (current_tag.tag == tag),
> +			     "%s called to perform a tag switch to the
> same tag\n", __func__);
> +		cvmx_warn_if(
> +			tag_type == CVMX_POW_TAG_TYPE_NULL,
> +			"%s called to perform a tag switch to NULL. Use
> cvmx_pow_tag_sw_null() instead\n",
> +			__func__);
> +		if ((wqp != cvmx_phys_to_ptr(0x80)) &&
> cvmx_pow_get_current_wqp())
> +			cvmx_warn_if(wqp != cvmx_pow_get_current_wqp(),
> +				     "%s passed WQE(%p) doesn't match
> the address in the POW(%p)\n",
> +				     __func__, wqp,
> cvmx_pow_get_current_wqp());
> +	}
> +	gxgrp = node;
> +	gxgrp = gxgrp << 8 | xgrp;
> +	wqp->word1.cn78xx.grp = gxgrp;
> +	wqp->word1.cn78xx.tag = tag;
> +	wqp->word1.cn78xx.tag_type = tag_type;
> +	CVMX_SYNCWS;
> +
> +	tag_req.u64 = 0;
> +	tag_req.s_cn78xx_other.op = CVMX_POW_TAG_OP_SWTAG_FULL;
> +	tag_req.s_cn78xx_other.type = tag_type;
> +	tag_req.s_cn78xx_other.grp = gxgrp;
> +	tag_req.s_cn78xx_other.wqp = cvmx_ptr_to_phys(wqp);
> +
> +	ptr.u64 = 0;
> +	ptr.s_cn78xx.mem_region = CVMX_IO_SEG;
> +	ptr.s_cn78xx.is_io = 1;
> +	ptr.s_cn78xx.did = CVMX_OCT_DID_TAG_SWTAG;
> +	ptr.s_cn78xx.node = node;
> +	ptr.s_cn78xx.tag = tag;
> +	cvmx_write_io(ptr.u64, tag_req.u64);
> +}
> +
> +/**
> + * Submits work to an SSO group on any OCI node.
> + * This function updates the work queue entry in DRAM to match
> + * the arguments given.
> + * Note that the tag provided is for the work queue entry submitted,
> + * and is unrelated to the tag that the core currently holds.
> + *
> + * @param wqp pointer to work queue entry to submit.
> + * This entry is updated to match the other parameters
> + * @param tag tag value to be assigned to work queue entry
> + * @param tag_type type of tag
> + * @param xgrp native CN78XX group in the range 0..255
> + * @param node The OCI node number for the target group
> + *
> + * When this function is called on a model prior to CN78XX, which
> does
> + * not support OCI nodes, the 'node' argument is ignored, and the
> 'xgrp'
> + * parameter is converted into 'qos' (the lower 3 bits) and 'grp'
> (the higher
> + * 5 bits), following the backward-compatibility scheme of
> translating
> + * between new and old style group numbers.
> + */
> +static inline void cvmx_pow_work_submit_node(cvmx_wqe_t *wqp, u32
> tag, cvmx_pow_tag_type_t tag_type,
> +					     u8 xgrp, u8 node)
> +{
> +	union cvmx_pow_tag_req_addr ptr;
> +	cvmx_pow_tag_req_t tag_req;
> +	u16 group;
> +
> +	if
> (cvmx_unlikely(!octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE))) {
> +		debug("ERROR: %s is supported on OCTEON3 only\n",
> __func__);
> +		return;
> +	}
> +	group = node;
> +	group = group << 8 | xgrp;
> +	wqp->word1.cn78xx.tag = tag;
> +	wqp->word1.cn78xx.tag_type = tag_type;
> +	wqp->word1.cn78xx.grp = group;
> +	CVMX_SYNCWS;
> +
> +	tag_req.u64 = 0;
> +	tag_req.s_cn78xx_other.op = CVMX_POW_TAG_OP_ADDWQ;
> +	tag_req.s_cn78xx_other.type = tag_type;
> +	tag_req.s_cn78xx_other.wqp = cvmx_ptr_to_phys(wqp);
> +	tag_req.s_cn78xx_other.grp = group;
> +
> +	ptr.u64 = 0;
> +	ptr.s_cn78xx.did = 0x66; // CVMX_OCT_DID_TAG_TAG6;
> +	ptr.s_cn78xx.mem_region = CVMX_IO_SEG;
> +	ptr.s_cn78xx.is_io = 1;
> +	ptr.s_cn78xx.node = node;
> +	ptr.s_cn78xx.tag = tag;
> +
> +	/* SYNC write to memory before the work submit.  This is
> necessary
> +	 ** as POW may read values from DRAM at this time */
> +	CVMX_SYNCWS;
> +	cvmx_write_io(ptr.u64, tag_req.u64);
> +}
> +
> +/**
> + * Executes the SSO SWTAG_DESCHED operation.
> + * This is similar to the cvmx_pow_tag_sw_desched() function, but
> + * uses linear (vs. unified group-qos) group index.
> + */
> +static inline void cvmx_pow_tag_sw_desched_node(cvmx_wqe_t *wqe, u32
> tag,
> +						cvmx_pow_tag_type_t
> tag_type, u8 xgrp, u64 no_sched,
> +						u8 node)
> +{
> +	union cvmx_pow_tag_req_addr ptr;
> +	cvmx_pow_tag_req_t tag_req;
> +	u16 group;
> +
> +	if
> (cvmx_unlikely(!octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE))) {
> +		debug("ERROR: %s is supported on OCTEON3 only\n",
> __func__);
> +		return;
> +	}
> +	/* Need to make sure any writes to the work queue entry are
> complete */
> +	CVMX_SYNCWS;
> +	/*
> +	 * Ensure that there is not a pending tag switch, as a tag
> switch cannot
> +	 * be started if a previous switch is still pending.
> +	 */
> +	cvmx_pow_tag_sw_wait();
> +
> +	if (CVMX_ENABLE_POW_CHECKS) {
> +		cvmx_pow_tag_info_t current_tag;
> +
> +		__cvmx_pow_warn_if_pending_switch(__func__);
> +		current_tag = cvmx_pow_get_current_tag();
> +		cvmx_warn_if(current_tag.tag_type ==
> CVMX_POW_TAG_TYPE_NULL_NULL,
> +			     "%s called with NULL_NULL tag\n",
> __func__);
> +		cvmx_warn_if(current_tag.tag_type ==
> CVMX_POW_TAG_TYPE_NULL,
> +			     "%s called with NULL tag. Deschedule not
> allowed from NULL state\n",
> +			     __func__);
> +		cvmx_warn_if((current_tag.tag_type !=
> CVMX_POW_TAG_TYPE_ATOMIC) &&
> +			     (tag_type != CVMX_POW_TAG_TYPE_ATOMIC),
> +			     "%s called where neither the before or
> after tag is ATOMIC\n",
> +			     __func__);
> +	}
> +	group = node;
> +	group = group << 8 | xgrp;
> +	wqe->word1.cn78xx.tag = tag;
> +	wqe->word1.cn78xx.tag_type = tag_type;
> +	wqe->word1.cn78xx.grp = group;
> +	CVMX_SYNCWS;
> +
> +	tag_req.u64 = 0;
> +	tag_req.s_cn78xx_other.op = CVMX_POW_TAG_OP_SWTAG_DESCH;
> +	tag_req.s_cn78xx_other.type = tag_type;
> +	tag_req.s_cn78xx_other.grp = group;
> +	tag_req.s_cn78xx_other.no_sched = no_sched;
> +
> +	ptr.u64 = 0;
> +	ptr.s.mem_region = CVMX_IO_SEG;
> +	ptr.s.is_io = 1;
> +	ptr.s.did = CVMX_OCT_DID_TAG_TAG3;
> +	ptr.s_cn78xx.node = node;
> +	ptr.s_cn78xx.tag = tag;
> +	cvmx_write_io(ptr.u64, tag_req.u64);
> +}
> +
> +/* Executes the UPD_WQP_GRP SSO operation.
> + *
> + * @param wqp  Pointer to the new work queue entry to switch to.
> + * @param xgrp SSO group in the range 0..255
> + *
> + * NOTE: The operation can be performed only on the local node.
> + */
> +static inline void cvmx_sso_update_wqp_group(cvmx_wqe_t *wqp, u8
> xgrp)
> +{
> +	union cvmx_pow_tag_req_addr addr;
> +	cvmx_pow_tag_req_t data;
> +	int node = cvmx_get_node_num();
> +	int group = node << 8 | xgrp;
> +
> +	if (!octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		debug("ERROR: %s is not supported on this chip)\n",
> __func__);
> +		return;
> +	}
> +	wqp->word1.cn78xx.grp = group;
> +	CVMX_SYNCWS;
> +
> +	data.u64 = 0;
> +	data.s_cn78xx_other.op = CVMX_POW_TAG_OP_UPDATE_WQP_GRP;
> +	data.s_cn78xx_other.grp = group;
> +	data.s_cn78xx_other.wqp = cvmx_ptr_to_phys(wqp);
> +
> +	addr.u64 = 0;
> +	addr.s_cn78xx.mem_region = CVMX_IO_SEG;
> +	addr.s_cn78xx.is_io = 1;
> +	addr.s_cn78xx.did = CVMX_OCT_DID_TAG_TAG1;
> +	addr.s_cn78xx.node = node;
> +	cvmx_write_io(addr.u64, data.u64);
> +}
> +
> +/*******************************************************************
> ***********/
> +/* Define usage of bits within the 32 bit tag
> values.                         */
> +/*******************************************************************
> ***********/
> +/*
> + * Number of bits of the tag used by software.  The SW bits
> + * are always a contiguous block of the high starting at bit 31.
> + * The hardware bits are always the low bits.  By default, the top 8
> bits
> + * of the tag are reserved for software, and the low 24 are set by
> the IPD unit.
> + */
> +#define CVMX_TAG_SW_BITS  (8)
> +#define CVMX_TAG_SW_SHIFT (32 - CVMX_TAG_SW_BITS)
> +
> +/* Below is the list of values for the top 8 bits of the tag. */
> +/*
> + * Tag values with top byte of this value are reserved for internal
> executive
> + * uses
> + */
> +#define CVMX_TAG_SW_BITS_INTERNAL 0x1
> +
> +/*
> + * The executive divides the remaining 24 bits as follows:
> + * the upper 8 bits (bits 23 - 16 of the tag) define a subgroup
> + * the lower 16 bits (bits 15 - 0 of the tag) define are the value
> with
> + * the subgroup. Note that this section describes the format of tags
> generated
> + * by software - refer to the hardware documentation for a
> description of the
> + * tags values generated by the packet input hardware.
> + * Subgroups are defined here
> + */
> +
> +/* Mask for the value portion of the tag */
> +#define CVMX_TAG_SUBGROUP_MASK	0xFFFF
> +#define CVMX_TAG_SUBGROUP_SHIFT 16
> +#define CVMX_TAG_SUBGROUP_PKO	0x1
> +
> +/* End of executive tag subgroup definitions */
> +
> +/* The remaining values software bit values 0x2 - 0xff are available
> + * for application use */
> +
> +/**
> + * This function creates a 32 bit tag value from the two values
> provided.
> + *
> + * @param sw_bits The upper bits (number depends on configuration)
> are set
> + *     to this value.  The remainder of bits are set by the hw_bits
> parameter.
> + * @param hw_bits The lower bits (number depends on configuration)
> are set
> + *     to this value.  The remainder of bits are set by the sw_bits
> parameter.
> + *
> + * @return 32 bit value of the combined hw and sw bits.
> + */
> +static inline u32 cvmx_pow_tag_compose(u64 sw_bits, u64 hw_bits)
> +{
> +	return (((sw_bits & cvmx_build_mask(CVMX_TAG_SW_BITS)) <<
> CVMX_TAG_SW_SHIFT) |
> +		(hw_bits & cvmx_build_mask(32 - CVMX_TAG_SW_BITS)));
> +}
> +
> +/**
> + * Extracts the bits allocated for software use from the tag
> + *
> + * @param tag    32 bit tag value
> + *
> + * @return N bit software tag value, where N is configurable with
> + *     the CVMX_TAG_SW_BITS define
> + */
> +static inline u32 cvmx_pow_tag_get_sw_bits(u64 tag)
> +{
> +	return ((tag >> (32 - CVMX_TAG_SW_BITS)) &
> cvmx_build_mask(CVMX_TAG_SW_BITS));
> +}
> +
> +/**
> + *
> + * Extracts the bits allocated for hardware use from the tag
> + *
> + * @param tag    32 bit tag value
> + *
> + * @return (32 - N) bit software tag value, where N is configurable
> with
> + *     the CVMX_TAG_SW_BITS define
> + */
> +static inline u32 cvmx_pow_tag_get_hw_bits(u64 tag)
> +{
> +	return (tag & cvmx_build_mask(32 - CVMX_TAG_SW_BITS));
> +}
> +
> +static inline u64 cvmx_sso3_get_wqe_count(int node)
> +{
> +	cvmx_sso_grpx_aq_cnt_t aq_cnt;
> +	unsigned int grp = 0;
> +	u64 cnt = 0;
> +
> +	for (grp = 0; grp < cvmx_sso_num_xgrp(); grp++) {
> +		aq_cnt.u64 = csr_rd_node(node,
> CVMX_SSO_GRPX_AQ_CNT(grp));
> +		cnt += aq_cnt.s.aq_cnt;
> +	}
> +	return cnt;
> +}
> +
> +static inline u64 cvmx_sso_get_total_wqe_count(void)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		int node = cvmx_get_node_num();
> +
> +		return cvmx_sso3_get_wqe_count(node);
> +	} else if (OCTEON_IS_MODEL(OCTEON_CN68XX)) {
> +		cvmx_sso_iq_com_cnt_t sso_iq_com_cnt;
> +
> +		sso_iq_com_cnt.u64 = csr_rd(CVMX_SSO_IQ_COM_CNT);
> +		return (sso_iq_com_cnt.s.iq_cnt);
> +	} else {
> +		cvmx_pow_iq_com_cnt_t pow_iq_com_cnt;
> +
> +		pow_iq_com_cnt.u64 = csr_rd(CVMX_POW_IQ_COM_CNT);
> +		return (pow_iq_com_cnt.s.iq_cnt);
> +	}
> +}
> +
> +/**
> + * Store the current POW internal state into the supplied
> + * buffer. It is recommended that you pass a buffer of at least
> + * 128KB. The format of the capture may change based on SDK
> + * version and Octeon chip.
> + *
> + * @param buffer Buffer to store capture into
> + * @param buffer_size The size of the supplied buffer
> + *
> + * @return Zero on success, negative on failure
> + */
> +int cvmx_pow_capture(void *buffer, int buffer_size);
> +
> +/**
> + * Dump a POW capture to the console in a human readable format.
> + *
> + * @param buffer POW capture from cvmx_pow_capture()
> + * @param buffer_size Size of the buffer
> + */
> +void cvmx_pow_display(void *buffer, int buffer_size);
> +
> +/**
> + * Return the number of POW entries supported by this chip
> + *
> + * @return Number of POW entries
> + */
> +int cvmx_pow_get_num_entries(void);
> +int cvmx_pow_get_dump_size(void);
> +
> +/**
> + * This will allocate count number of SSO groups on the specified
> node to the
> + * calling application. These groups will be for exclusive use of
> the
> + * application until they are freed.
> + * @param node The numa node for the allocation.
> + * @param base_group Pointer to the initial group, -1 to allocate
> anywhere.
> + * @param count  The number of consecutive groups to allocate.
> + * @return 0 on success and -1 on failure.
> + */
> +int cvmx_sso_reserve_group_range(int node, int *base_group, int
> count);
> +#define cvmx_sso_allocate_group_range cvmx_sso_reserve_group_range
> +int cvmx_sso_reserve_group(int node);
> +#define cvmx_sso_allocate_group cvmx_sso_reserve_group
> +int cvmx_sso_release_group_range(int node, int base_group, int
> count);
> +int cvmx_sso_release_group(int node, int group);
> +
> +/**
> + * Show integrated SSO configuration.
> + *
> + * @param node	   node number
> + */
> +int cvmx_sso_config_dump(unsigned int node);
> +
> +/**
> + * Show integrated SSO statistics.
> + *
> + * @param node	   node number
> + */
> +int cvmx_sso_stats_dump(unsigned int node);
> +
> +/**
> + * Clear integrated SSO statistics.
> + *
> + * @param node	   node number
> + */
> +int cvmx_sso_stats_clear(unsigned int node);
> +
> +/**
> + * Show SSO core-group affinity and priority per node (multi-node
> systems)
> + */
> +void cvmx_pow_mask_priority_dump_node(unsigned int node, struct
> cvmx_coremask *avail_coremask);
> +
> +/**
> + * Show POW/SSO core-group affinity and priority (legacy, single-
> node systems)
> + */
> +static inline void cvmx_pow_mask_priority_dump(struct cvmx_coremask
> *avail_coremask)
> +{
> +	cvmx_pow_mask_priority_dump_node(0 /*node */, avail_coremask);
> +}
> +
> +/**
> + * Show SSO performance counters (multi-node systems)
> + */
> +void cvmx_pow_show_perf_counters_node(unsigned int node);
> +
> +/**
> + * Show POW/SSO performance counters (legacy, single-node systems)
> + */
> +static inline void cvmx_pow_show_perf_counters(void)
> +{
> +	cvmx_pow_show_perf_counters_node(0 /*node */);
> +}
> +
> +#endif /* __CVMX_POW_H__ */
> diff --git a/arch/mips/mach-octeon/include/mach/cvmx-qlm.h
> b/arch/mips/mach-octeon/include/mach/cvmx-qlm.h
> new file mode 100644
> index 000000000000..19915eb82c51
> --- /dev/null
> +++ b/arch/mips/mach-octeon/include/mach/cvmx-qlm.h
> @@ -0,0 +1,304 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +/*
> + * Copyright (C) 2020 Marvell International Ltd.
> + */
> +
> +#ifndef __CVMX_QLM_H__
> +#define __CVMX_QLM_H__
> +
> +/*
> + * Interface 0 on the 78xx can be connected to qlm 0 or qlm 2. When
> interface
> + * 0 is connected to qlm 0, this macro must be set to 0. When
> interface 0 is
> + * connected to qlm 2, this macro must be set to 1.
> + */
> +#define MUX_78XX_IFACE0 0
> +
> +/*
> + * Interface 1 on the 78xx can be connected to qlm 1 or qlm 3. When
> interface
> + * 1 is connected to qlm 1, this macro must be set to 0. When
> interface 1 is
> + * connected to qlm 3, this macro must be set to 1.
> + */
> +#define MUX_78XX_IFACE1 0
> +
> +/* Uncomment this line to print QLM JTAG state */
> +/* #define CVMX_QLM_DUMP_STATE 1 */
> +
> +typedef struct {
> +	const char *name;
> +	int stop_bit;
> +	int start_bit;
> +} __cvmx_qlm_jtag_field_t;
> +
> +/**
> + * Return the number of QLMs supported by the chip
> + *
> + * @return  Number of QLMs
> + */
> +int cvmx_qlm_get_num(void);
> +
> +/**
> + * Return the qlm number based on the interface
> + *
> + * @param xiface  Interface to look
> + */
> +int cvmx_qlm_interface(int xiface);
> +
> +/**
> + * Return the qlm number based for a port in the interface
> + *
> + * @param xiface  interface to look up
> + * @param index  index in an interface
> + *
> + * @return the qlm number based on the xiface
> + */
> +int cvmx_qlm_lmac(int xiface, int index);
> +
> +/**
> + * Return if only DLM5/DLM6/DLM5+DLM6 is used by BGX
> + *
> + * @param BGX  BGX to search for.
> + *
> + * @return muxes used 0 = DLM5+DLM6, 1 = DLM5, 2 = DLM6.
> + */
> +int cvmx_qlm_mux_interface(int bgx);
> +
> +/**
> + * Return number of lanes for a given qlm
> + *
> + * @param qlm QLM block to query
> + *
> + * @return  Number of lanes
> + */
> +int cvmx_qlm_get_lanes(int qlm);
> +
> +/**
> + * Get the QLM JTAG fields based on Octeon model on the supported
> chips.
> + *
> + * @return  qlm_jtag_field_t structure
> + */
> +const __cvmx_qlm_jtag_field_t *cvmx_qlm_jtag_get_field(void);
> +
> +/**
> + * Get the QLM JTAG length by going through qlm_jtag_field for each
> + * Octeon model that is supported
> + *
> + * @return return the length.
> + */
> +int cvmx_qlm_jtag_get_length(void);
> +
> +/**
> + * Initialize the QLM layer
> + */
> +void cvmx_qlm_init(void);
> +
> +/**
> + * Get a field in a QLM JTAG chain
> + *
> + * @param qlm    QLM to get
> + * @param lane   Lane in QLM to get
> + * @param name   String name of field
> + *
> + * @return JTAG field value
> + */
> +u64 cvmx_qlm_jtag_get(int qlm, int lane, const char *name);
> +
> +/**
> + * Set a field in a QLM JTAG chain
> + *
> + * @param qlm    QLM to set
> + * @param lane   Lane in QLM to set, or -1 for all lanes
> + * @param name   String name of field
> + * @param value  Value of the field
> + */
> +void cvmx_qlm_jtag_set(int qlm, int lane, const char *name, u64
> value);
> +
> +/**
> + * Errata G-16094: QLM Gen2 Equalizer Default Setting Change.
> + * CN68XX pass 1.x and CN66XX pass 1.x QLM tweak. This function
> tweaks the
> + * JTAG setting for a QLMs to run better at 5 and 6.25Ghz.
> + */
> +void __cvmx_qlm_speed_tweak(void);
> +
> +/**
> + * Errata G-16174: QLM Gen2 PCIe IDLE DAC change.
> + * CN68XX pass 1.x, CN66XX pass 1.x and CN63XX pass 1.0-2.2 QLM
> tweak.
> + * This function tweaks the JTAG setting for a QLMs for PCIe to run
> better.
> + */
> +void __cvmx_qlm_pcie_idle_dac_tweak(void);
> +
> +void __cvmx_qlm_pcie_cfg_rxd_set_tweak(int qlm, int lane);
> +
> +/**
> + * Get the speed (Gbaud) of the QLM in Mhz.
> + *
> + * @param qlm    QLM to examine
> + *
> + * @return Speed in Mhz
> + */
> +int cvmx_qlm_get_gbaud_mhz(int qlm);
> +/**
> + * Get the speed (Gbaud) of the QLM in Mhz on specific node.
> + *
> + * @param node   Target QLM node
> + * @param qlm    QLM to examine
> + *
> + * @return Speed in Mhz
> + */
> +int cvmx_qlm_get_gbaud_mhz_node(int node, int qlm);
> +
> +enum cvmx_qlm_mode {
> +	CVMX_QLM_MODE_DISABLED = -1,
> +	CVMX_QLM_MODE_SGMII = 1,
> +	CVMX_QLM_MODE_XAUI,
> +	CVMX_QLM_MODE_RXAUI,
> +	CVMX_QLM_MODE_PCIE,	/* gen3 / gen2 / gen1 */
> +	CVMX_QLM_MODE_PCIE_1X2, /* 1x2 gen2 / gen1 */
> +	CVMX_QLM_MODE_PCIE_2X1, /* 2x1 gen2 / gen1 */
> +	CVMX_QLM_MODE_PCIE_1X1, /* 1x1 gen2 / gen1 */
> +	CVMX_QLM_MODE_SRIO_1X4, /* 1x4 short / long */
> +	CVMX_QLM_MODE_SRIO_2X2, /* 2x2 short / long */
> +	CVMX_QLM_MODE_SRIO_4X1, /* 4x1 short / long */
> +	CVMX_QLM_MODE_ILK,
> +	CVMX_QLM_MODE_QSGMII,
> +	CVMX_QLM_MODE_SGMII_SGMII,
> +	CVMX_QLM_MODE_SGMII_DISABLED,
> +	CVMX_QLM_MODE_DISABLED_SGMII,
> +	CVMX_QLM_MODE_SGMII_QSGMII,
> +	CVMX_QLM_MODE_QSGMII_QSGMII,
> +	CVMX_QLM_MODE_QSGMII_DISABLED,
> +	CVMX_QLM_MODE_DISABLED_QSGMII,
> +	CVMX_QLM_MODE_QSGMII_SGMII,
> +	CVMX_QLM_MODE_RXAUI_1X2,
> +	CVMX_QLM_MODE_SATA_2X1,
> +	CVMX_QLM_MODE_XLAUI,
> +	CVMX_QLM_MODE_XFI,
> +	CVMX_QLM_MODE_10G_KR,
> +	CVMX_QLM_MODE_40G_KR4,
> +	CVMX_QLM_MODE_PCIE_1X8, /* 1x8 gen3 / gen2 / gen1 */
> +	CVMX_QLM_MODE_RGMII_SGMII,
> +	CVMX_QLM_MODE_RGMII_XFI,
> +	CVMX_QLM_MODE_RGMII_10G_KR,
> +	CVMX_QLM_MODE_RGMII_RXAUI,
> +	CVMX_QLM_MODE_RGMII_XAUI,
> +	CVMX_QLM_MODE_RGMII_XLAUI,
> +	CVMX_QLM_MODE_RGMII_40G_KR4,
> +	CVMX_QLM_MODE_MIXED,		/* BGX2 is mixed mode,
> DLM5(SGMII) & DLM6(XFI) */
> +	CVMX_QLM_MODE_SGMII_2X1,	/* Configure BGX2 separate for DLM5 &
> DLM6 */
> +	CVMX_QLM_MODE_10G_KR_1X2,	/* Configure BGX2 separate for DLM5 &
> DLM6 */
> +	CVMX_QLM_MODE_XFI_1X2,		/* Configure BGX2 separate
> for DLM5 & DLM6 */
> +	CVMX_QLM_MODE_RGMII_SGMII_1X1,	/* Configure BGX2, applies to
> DLM5 */
> +	CVMX_QLM_MODE_RGMII_SGMII_2X1,	/* Configure BGX2, applies to
> DLM6 */
> +	CVMX_QLM_MODE_RGMII_10G_KR_1X1, /* Configure BGX2, applies to
> DLM6 */
> +	CVMX_QLM_MODE_RGMII_XFI_1X1,	/* Configure BGX2, applies to
> DLM6 */
> +	CVMX_QLM_MODE_SDL,		/* RMAC Pipe */
> +	CVMX_QLM_MODE_CPRI,		/* RMAC */
> +	CVMX_QLM_MODE_OCI
> +};
> +
> +enum cvmx_gmx_inf_mode {
> +	CVMX_GMX_INF_MODE_DISABLED = 0,
> +	CVMX_GMX_INF_MODE_SGMII = 1,  /* Other interface can be SGMII
> or QSGMII */
> +	CVMX_GMX_INF_MODE_QSGMII = 2, /* Other interface can be SGMII
> or QSGMII */
> +	CVMX_GMX_INF_MODE_RXAUI = 3,  /* Only interface 0, interface 1
> must be DISABLED */
> +};
> +
> +/**
> + * Eye diagram captures are stored in the following structure
> + */
> +typedef struct {
> +	int width;	   /* Width in the x direction (time) */
> +	int height;	   /* Height in the y direction (voltage) */
> +	u32 data[64][128]; /* Error count at location, saturates as max
> */
> +} cvmx_qlm_eye_t;
> +
> +/**
> + * These apply to DLM1 and DLM2 if its not in SATA mode
> + * Manual refers to lanes as follows:
> + *  DML 0 lane 0 == GSER0 lane 0
> + *  DML 0 lane 1 == GSER0 lane 1
> + *  DML 1 lane 2 == GSER1 lane 0
> + *  DML 1 lane 3 == GSER1 lane 1
> + *  DML 2 lane 4 == GSER2 lane 0
> + *  DML 2 lane 5 == GSER2 lane 1
> + */
> +enum cvmx_pemx_cfg_mode {
> +	CVMX_PEM_MD_GEN2_2LANE = 0, /* Valid for PEM0(DLM1), PEM1(DLM2)
> */
> +	CVMX_PEM_MD_GEN2_1LANE = 1, /* Valid for PEM0(DLM1.0),
> PEM1(DLM1.1,DLM2.0), PEM2(DLM2.1) */
> +	CVMX_PEM_MD_GEN2_4LANE = 2, /* Valid for PEM0(DLM1-2) */
> +	/* Reserved */
> +	CVMX_PEM_MD_GEN1_2LANE = 4, /* Valid for PEM0(DLM1), PEM1(DLM2)
> */
> +	CVMX_PEM_MD_GEN1_1LANE = 5, /* Valid for PEM0(DLM1.0),
> PEM1(DLM1.1,DLM2.0), PEM2(DLM2.1) */
> +	CVMX_PEM_MD_GEN1_4LANE = 6, /* Valid for PEM0(DLM1-2) */
> +	/* Reserved */
> +};
> +
> +/*
> + * Read QLM and return mode.
> + */
> +enum cvmx_qlm_mode cvmx_qlm_get_mode(int qlm);
> +enum cvmx_qlm_mode cvmx_qlm_get_mode_cn78xx(int node, int qlm);
> +enum cvmx_qlm_mode cvmx_qlm_get_dlm_mode(int dlm_mode, int
> interface);
> +void __cvmx_qlm_set_mult(int qlm, int baud_mhz, int old_multiplier);
> +
> +void cvmx_qlm_display_registers(int qlm);
> +
> +int cvmx_qlm_measure_clock(int qlm);
> +
> +/**
> + * Measure the reference clock of a QLM on a multi-node setup
> + *
> + * @param node   node to measure
> + * @param qlm    QLM to measure
> + *
> + * @return Clock rate in Hz
> + */
> +int cvmx_qlm_measure_clock_node(int node, int qlm);
> +
> +/*
> + * Perform RX equalization on a QLM
> + *
> + * @param node	Node the QLM is on
> + * @param qlm	QLM to perform RX equalization on
> + * @param lane	Lane to use, or -1 for all lanes
> + *
> + * @return Zero on success, negative if any lane failed RX
> equalization
> + */
> +int __cvmx_qlm_rx_equalization(int node, int qlm, int lane);
> +
> +/**
> + * Errata GSER-27882 -GSER 10GBASE-KR Transmit Equalizer
> + * Training may not update PHY Tx Taps. This function is not static
> + * so we can share it with BGX KR
> + *
> + * @param node	Node to apply errata workaround
> + * @param qlm	QLM to apply errata workaround
> + * @param lane	Lane to apply the errata
> + */
> +int cvmx_qlm_gser_errata_27882(int node, int qlm, int lane);
> +
> +void cvmx_qlm_gser_errata_25992(int node, int qlm);
> +
> +#ifdef CVMX_DUMP_GSER
> +/**
> + * Dump GSER configuration for node 0
> + */
> +int cvmx_dump_gser_config(unsigned int gser);
> +/**
> + * Dump GSER status for node 0
> + */
> +int cvmx_dump_gser_status(unsigned int gser);
> +/**
> + * Dump GSER configuration
> + */
> +int cvmx_dump_gser_config_node(unsigned int node, unsigned int
> gser);
> +/**
> + * Dump GSER status
> + */
> +int cvmx_dump_gser_status_node(unsigned int node, unsigned int
> gser);
> +#endif
> +
> +int cvmx_qlm_eye_display(int node, int qlm, int qlm_lane, int
> format, const cvmx_qlm_eye_t *eye);
> +
> +void cvmx_prbs_process_cmd(int node, int qlm, int mode);
> +
> +#endif /* __CVMX_QLM_H__ */
> diff --git a/arch/mips/mach-octeon/include/mach/cvmx-scratch.h
> b/arch/mips/mach-octeon/include/mach/cvmx-scratch.h
> new file mode 100644
> index 000000000000..d567a8453b7a
> --- /dev/null
> +++ b/arch/mips/mach-octeon/include/mach/cvmx-scratch.h
> @@ -0,0 +1,113 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +/*
> + * Copyright (C) 2020 Marvell International Ltd.
> + *
> + * This file provides support for the processor local scratch
> memory.
> + * Scratch memory is byte addressable - all addresses are byte
> addresses.
> + */
> +
> +#ifndef __CVMX_SCRATCH_H__
> +#define __CVMX_SCRATCH_H__
> +
> +/* Note: This define must be a long, not a long long in order to
> compile
> +	without warnings for both 32bit and 64bit. */
> +#define CVMX_SCRATCH_BASE (-32768l) /* 0xffffffffffff8000 */
> +
> +/* Scratch line for LMTST/LMTDMA on Octeon3 models */
> +#ifdef CVMX_CAVIUM_OCTEON3
> +#define CVMX_PKO_LMTLINE 2ull
> +#endif
> +
> +/**
> + * Reads an 8 bit value from the processor local scratchpad memory.
> + *
> + * @param address byte address to read from
> + *
> + * @return value read
> + */
> +static inline u8 cvmx_scratch_read8(u64 address)
> +{
> +	return *CASTPTR(volatile u8, CVMX_SCRATCH_BASE + address);
> +}
> +
> +/**
> + * Reads a 16 bit value from the processor local scratchpad memory.
> + *
> + * @param address byte address to read from
> + *
> + * @return value read
> + */
> +static inline u16 cvmx_scratch_read16(u64 address)
> +{
> +	return *CASTPTR(volatile u16, CVMX_SCRATCH_BASE + address);
> +}
> +
> +/**
> + * Reads a 32 bit value from the processor local scratchpad memory.
> + *
> + * @param address byte address to read from
> + *
> + * @return value read
> + */
> +static inline u32 cvmx_scratch_read32(u64 address)
> +{
> +	return *CASTPTR(volatile u32, CVMX_SCRATCH_BASE + address);
> +}
> +
> +/**
> + * Reads a 64 bit value from the processor local scratchpad memory.
> + *
> + * @param address byte address to read from
> + *
> + * @return value read
> + */
> +static inline u64 cvmx_scratch_read64(u64 address)
> +{
> +	return *CASTPTR(volatile u64, CVMX_SCRATCH_BASE + address);
> +}
> +
> +/**
> + * Writes an 8 bit value to the processor local scratchpad memory.
> + *
> + * @param address byte address to write to
> + * @param value   value to write
> + */
> +static inline void cvmx_scratch_write8(u64 address, u64 value)
> +{
> +	*CASTPTR(volatile u8, CVMX_SCRATCH_BASE + address) = (u8)value;
> +}
> +
> +/**
> + * Writes a 32 bit value to the processor local scratchpad memory.
> + *
> + * @param address byte address to write to
> + * @param value   value to write
> + */
> +static inline void cvmx_scratch_write16(u64 address, u64 value)
> +{
> +	*CASTPTR(volatile u16, CVMX_SCRATCH_BASE + address) =
> (u16)value;
> +}
> +
> +/**
> + * Writes a 16 bit value to the processor local scratchpad memory.
> + *
> + * @param address byte address to write to
> + * @param value   value to write
> + */
> +static inline void cvmx_scratch_write32(u64 address, u64 value)
> +{
> +	*CASTPTR(volatile u32, CVMX_SCRATCH_BASE + address) =
> (u32)value;
> +}
> +
> +/**
> + * Writes a 64 bit value to the processor local scratchpad memory.
> + *
> + * @param address byte address to write to
> + * @param value   value to write
> + */
> +static inline void cvmx_scratch_write64(u64 address, u64 value)
> +{
> +	*CASTPTR(volatile u64, CVMX_SCRATCH_BASE + address) = value;
> +}
> +
> +#endif /* __CVMX_SCRATCH_H__ */
> diff --git a/arch/mips/mach-octeon/include/mach/cvmx-wqe.h
> b/arch/mips/mach-octeon/include/mach/cvmx-wqe.h
> new file mode 100644
> index 000000000000..c9e3c8312a65
> --- /dev/null
> +++ b/arch/mips/mach-octeon/include/mach/cvmx-wqe.h
> @@ -0,0 +1,1462 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +/*
> + * Copyright (C) 2020 Marvell International Ltd.
> + *
> + * This header file defines the work queue entry (wqe) data
> structure.
> + * Since this is a commonly used structure that depends on
> structures
> + * from several hardware blocks, those definitions have been placed
> + * in this file to create a single point of definition of the wqe
> + * format.
> + * Data structures are still named according to the block that they
> + * relate to.
> + */
> +
> +#ifndef __CVMX_WQE_H__
> +#define __CVMX_WQE_H__
> +
> +#include "cvmx-packet.h"
> +#include "cvmx-csr-enums.h"
> +#include "cvmx-pki-defs.h"
> +#include "cvmx-pip-defs.h"
> +#include "octeon-feature.h"
> +
> +#define OCT_TAG_TYPE_STRING(x)					
> 	\
> +	(((x) == CVMX_POW_TAG_TYPE_ORDERED) ?				
> \
> +	 "ORDERED" :							
> \
> +	 (((x) == CVMX_POW_TAG_TYPE_ATOMIC) ?				
> \
> +	  "ATOMIC" :							
> \
> +	  (((x) == CVMX_POW_TAG_TYPE_NULL) ? "NULL" : "NULL_NULL")))
> +
> +/* Error levels in WQE WORD2 (ERRLEV).*/
> +#define PKI_ERRLEV_E__RE_M 0x0
> +#define PKI_ERRLEV_E__LA_M 0x1
> +#define PKI_ERRLEV_E__LB_M 0x2
> +#define PKI_ERRLEV_E__LC_M 0x3
> +#define PKI_ERRLEV_E__LD_M 0x4
> +#define PKI_ERRLEV_E__LE_M 0x5
> +#define PKI_ERRLEV_E__LF_M 0x6
> +#define PKI_ERRLEV_E__LG_M 0x7
> +
> +enum cvmx_pki_errlevel {
> +	CVMX_PKI_ERRLEV_E_RE = PKI_ERRLEV_E__RE_M,
> +	CVMX_PKI_ERRLEV_E_LA = PKI_ERRLEV_E__LA_M,
> +	CVMX_PKI_ERRLEV_E_LB = PKI_ERRLEV_E__LB_M,
> +	CVMX_PKI_ERRLEV_E_LC = PKI_ERRLEV_E__LC_M,
> +	CVMX_PKI_ERRLEV_E_LD = PKI_ERRLEV_E__LD_M,
> +	CVMX_PKI_ERRLEV_E_LE = PKI_ERRLEV_E__LE_M,
> +	CVMX_PKI_ERRLEV_E_LF = PKI_ERRLEV_E__LF_M,
> +	CVMX_PKI_ERRLEV_E_LG = PKI_ERRLEV_E__LG_M
> +};
> +
> +#define CVMX_PKI_ERRLEV_MAX BIT(3) /* The size of WORD2:ERRLEV
> field.*/
> +
> +/* Error code in WQE WORD2 (OPCODE).*/
> +#define CVMX_PKI_OPCODE_RE_NONE	      0x0
> +#define CVMX_PKI_OPCODE_RE_PARTIAL    0x1
> +#define CVMX_PKI_OPCODE_RE_JABBER     0x2
> +#define CVMX_PKI_OPCODE_RE_FCS	      0x7
> +#define CVMX_PKI_OPCODE_RE_FCS_RCV    0x8
> +#define CVMX_PKI_OPCODE_RE_TERMINATE  0x9
> +#define CVMX_PKI_OPCODE_RE_RX_CTL     0xb
> +#define CVMX_PKI_OPCODE_RE_SKIP	      0xc
> +#define CVMX_PKI_OPCODE_RE_DMAPKT     0xf
> +#define CVMX_PKI_OPCODE_RE_PKIPAR     0x13
> +#define CVMX_PKI_OPCODE_RE_PKIPCAM    0x14
> +#define CVMX_PKI_OPCODE_RE_MEMOUT     0x15
> +#define CVMX_PKI_OPCODE_RE_BUFS_OFLOW 0x16
> +#define CVMX_PKI_OPCODE_L2_FRAGMENT   0x20
> +#define CVMX_PKI_OPCODE_L2_OVERRUN    0x21
> +#define CVMX_PKI_OPCODE_L2_PFCS	      0x22
> +#define CVMX_PKI_OPCODE_L2_PUNY	      0x23
> +#define CVMX_PKI_OPCODE_L2_MAL	      0x24
> +#define CVMX_PKI_OPCODE_L2_OVERSIZE   0x25
> +#define CVMX_PKI_OPCODE_L2_UNDERSIZE  0x26
> +#define CVMX_PKI_OPCODE_L2_LENMISM    0x27
> +#define CVMX_PKI_OPCODE_IP_NOT	      0x41
> +#define CVMX_PKI_OPCODE_IP_CHK	      0x42
> +#define CVMX_PKI_OPCODE_IP_MAL	      0x43
> +#define CVMX_PKI_OPCODE_IP_MALD	      0x44
> +#define CVMX_PKI_OPCODE_IP_HOP	      0x45
> +#define CVMX_PKI_OPCODE_L4_MAL	      0x61
> +#define CVMX_PKI_OPCODE_L4_CHK	      0x62
> +#define CVMX_PKI_OPCODE_L4_LEN	      0x63
> +#define CVMX_PKI_OPCODE_L4_PORT	      0x64
> +#define CVMX_PKI_OPCODE_TCP_FLAG      0x65
> +
> +#define CVMX_PKI_OPCODE_MAX BIT(8) /* The size of WORD2:OPCODE
> field.*/
> +
> +/* Layer types in pki */
> +#define CVMX_PKI_LTYPE_E_NONE_M	      0x0
> +#define CVMX_PKI_LTYPE_E_ENET_M	      0x1
> +#define CVMX_PKI_LTYPE_E_VLAN_M	      0x2
> +#define CVMX_PKI_LTYPE_E_SNAP_PAYLD_M 0x5
> +#define CVMX_PKI_LTYPE_E_ARP_M	      0x6
> +#define CVMX_PKI_LTYPE_E_RARP_M	      0x7
> +#define CVMX_PKI_LTYPE_E_IP4_M	      0x8
> +#define CVMX_PKI_LTYPE_E_IP4_OPT_M    0x9
> +#define CVMX_PKI_LTYPE_E_IP6_M	      0xA
> +#define CVMX_PKI_LTYPE_E_IP6_OPT_M    0xB
> +#define CVMX_PKI_LTYPE_E_IPSEC_ESP_M  0xC
> +#define CVMX_PKI_LTYPE_E_IPFRAG_M     0xD
> +#define CVMX_PKI_LTYPE_E_IPCOMP_M     0xE
> +#define CVMX_PKI_LTYPE_E_TCP_M	      0x10
> +#define CVMX_PKI_LTYPE_E_UDP_M	      0x11
> +#define CVMX_PKI_LTYPE_E_SCTP_M	      0x12
> +#define CVMX_PKI_LTYPE_E_UDP_VXLAN_M  0x13
> +#define CVMX_PKI_LTYPE_E_GRE_M	      0x14
> +#define CVMX_PKI_LTYPE_E_NVGRE_M      0x15
> +#define CVMX_PKI_LTYPE_E_GTP_M	      0x16
> +#define CVMX_PKI_LTYPE_E_SW28_M	      0x1C
> +#define CVMX_PKI_LTYPE_E_SW29_M	      0x1D
> +#define CVMX_PKI_LTYPE_E_SW30_M	      0x1E
> +#define CVMX_PKI_LTYPE_E_SW31_M	      0x1F
> +
> +enum cvmx_pki_layer_type {
> +	CVMX_PKI_LTYPE_E_NONE = CVMX_PKI_LTYPE_E_NONE_M,
> +	CVMX_PKI_LTYPE_E_ENET = CVMX_PKI_LTYPE_E_ENET_M,
> +	CVMX_PKI_LTYPE_E_VLAN = CVMX_PKI_LTYPE_E_VLAN_M,
> +	CVMX_PKI_LTYPE_E_SNAP_PAYLD = CVMX_PKI_LTYPE_E_SNAP_PAYLD_M,
> +	CVMX_PKI_LTYPE_E_ARP = CVMX_PKI_LTYPE_E_ARP_M,
> +	CVMX_PKI_LTYPE_E_RARP = CVMX_PKI_LTYPE_E_RARP_M,
> +	CVMX_PKI_LTYPE_E_IP4 = CVMX_PKI_LTYPE_E_IP4_M,
> +	CVMX_PKI_LTYPE_E_IP4_OPT = CVMX_PKI_LTYPE_E_IP4_OPT_M,
> +	CVMX_PKI_LTYPE_E_IP6 = CVMX_PKI_LTYPE_E_IP6_M,
> +	CVMX_PKI_LTYPE_E_IP6_OPT = CVMX_PKI_LTYPE_E_IP6_OPT_M,
> +	CVMX_PKI_LTYPE_E_IPSEC_ESP = CVMX_PKI_LTYPE_E_IPSEC_ESP_M,
> +	CVMX_PKI_LTYPE_E_IPFRAG = CVMX_PKI_LTYPE_E_IPFRAG_M,
> +	CVMX_PKI_LTYPE_E_IPCOMP = CVMX_PKI_LTYPE_E_IPCOMP_M,
> +	CVMX_PKI_LTYPE_E_TCP = CVMX_PKI_LTYPE_E_TCP_M,
> +	CVMX_PKI_LTYPE_E_UDP = CVMX_PKI_LTYPE_E_UDP_M,
> +	CVMX_PKI_LTYPE_E_SCTP = CVMX_PKI_LTYPE_E_SCTP_M,
> +	CVMX_PKI_LTYPE_E_UDP_VXLAN = CVMX_PKI_LTYPE_E_UDP_VXLAN_M,
> +	CVMX_PKI_LTYPE_E_GRE = CVMX_PKI_LTYPE_E_GRE_M,
> +	CVMX_PKI_LTYPE_E_NVGRE = CVMX_PKI_LTYPE_E_NVGRE_M,
> +	CVMX_PKI_LTYPE_E_GTP = CVMX_PKI_LTYPE_E_GTP_M,
> +	CVMX_PKI_LTYPE_E_SW28 = CVMX_PKI_LTYPE_E_SW28_M,
> +	CVMX_PKI_LTYPE_E_SW29 = CVMX_PKI_LTYPE_E_SW29_M,
> +	CVMX_PKI_LTYPE_E_SW30 = CVMX_PKI_LTYPE_E_SW30_M,
> +	CVMX_PKI_LTYPE_E_SW31 = CVMX_PKI_LTYPE_E_SW31_M,
> +	CVMX_PKI_LTYPE_E_MAX = CVMX_PKI_LTYPE_E_SW31
> +};
> +
> +typedef union {
> +	u64 u64;
> +	struct {
> +		u64 ptr_vlan : 8;
> +		u64 ptr_layer_g : 8;
> +		u64 ptr_layer_f : 8;
> +		u64 ptr_layer_e : 8;
> +		u64 ptr_layer_d : 8;
> +		u64 ptr_layer_c : 8;
> +		u64 ptr_layer_b : 8;
> +		u64 ptr_layer_a : 8;
> +	};
> +} cvmx_pki_wqe_word4_t;
> +
> +/**
> + * HW decode / err_code in work queue entry
> + */
> +typedef union {
> +	u64 u64;
> +	struct {
> +		u64 bufs : 8;
> +		u64 ip_offset : 8;
> +		u64 vlan_valid : 1;
> +		u64 vlan_stacked : 1;
> +		u64 unassigned : 1;
> +		u64 vlan_cfi : 1;
> +		u64 vlan_id : 12;
> +		u64 varies : 12;
> +		u64 dec_ipcomp : 1;
> +		u64 tcp_or_udp : 1;
> +		u64 dec_ipsec : 1;
> +		u64 is_v6 : 1;
> +		u64 software : 1;
> +		u64 L4_error : 1;
> +		u64 is_frag : 1;
> +		u64 IP_exc : 1;
> +		u64 is_bcast : 1;
> +		u64 is_mcast : 1;
> +		u64 not_IP : 1;
> +		u64 rcv_error : 1;
> +		u64 err_code : 8;
> +	} s;
> +	struct {
> +		u64 bufs : 8;
> +		u64 ip_offset : 8;
> +		u64 vlan_valid : 1;
> +		u64 vlan_stacked : 1;
> +		u64 unassigned : 1;
> +		u64 vlan_cfi : 1;
> +		u64 vlan_id : 12;
> +		u64 port : 12;
> +		u64 dec_ipcomp : 1;
> +		u64 tcp_or_udp : 1;
> +		u64 dec_ipsec : 1;
> +		u64 is_v6 : 1;
> +		u64 software : 1;
> +		u64 L4_error : 1;
> +		u64 is_frag : 1;
> +		u64 IP_exc : 1;
> +		u64 is_bcast : 1;
> +		u64 is_mcast : 1;
> +		u64 not_IP : 1;
> +		u64 rcv_error : 1;
> +		u64 err_code : 8;
> +	} s_cn68xx;
> +	struct {
> +		u64 bufs : 8;
> +		u64 ip_offset : 8;
> +		u64 vlan_valid : 1;
> +		u64 vlan_stacked : 1;
> +		u64 unassigned : 1;
> +		u64 vlan_cfi : 1;
> +		u64 vlan_id : 12;
> +		u64 pr : 4;
> +		u64 unassigned2a : 4;
> +		u64 unassigned2 : 4;
> +		u64 dec_ipcomp : 1;
> +		u64 tcp_or_udp : 1;
> +		u64 dec_ipsec : 1;
> +		u64 is_v6 : 1;
> +		u64 software : 1;
> +		u64 L4_error : 1;
> +		u64 is_frag : 1;
> +		u64 IP_exc : 1;
> +		u64 is_bcast : 1;
> +		u64 is_mcast : 1;
> +		u64 not_IP : 1;
> +		u64 rcv_error : 1;
> +		u64 err_code : 8;
> +	} s_cn38xx;
> +	struct {
> +		u64 unused1 : 16;
> +		u64 vlan : 16;
> +		u64 unused2 : 32;
> +	} svlan;
> +	struct {
> +		u64 bufs : 8;
> +		u64 unused : 8;
> +		u64 vlan_valid : 1;
> +		u64 vlan_stacked : 1;
> +		u64 unassigned : 1;
> +		u64 vlan_cfi : 1;
> +		u64 vlan_id : 12;
> +		u64 varies : 12;
> +		u64 unassigned2 : 4;
> +		u64 software : 1;
> +		u64 unassigned3 : 1;
> +		u64 is_rarp : 1;
> +		u64 is_arp : 1;
> +		u64 is_bcast : 1;
> +		u64 is_mcast : 1;
> +		u64 not_IP : 1;
> +		u64 rcv_error : 1;
> +		u64 err_code : 8;
> +	} snoip;
> +	struct {
> +		u64 bufs : 8;
> +		u64 unused : 8;
> +		u64 vlan_valid : 1;
> +		u64 vlan_stacked : 1;
> +		u64 unassigned : 1;
> +		u64 vlan_cfi : 1;
> +		u64 vlan_id : 12;
> +		u64 port : 12;
> +		u64 unassigned2 : 4;
> +		u64 software : 1;
> +		u64 unassigned3 : 1;
> +		u64 is_rarp : 1;
> +		u64 is_arp : 1;
> +		u64 is_bcast : 1;
> +		u64 is_mcast : 1;
> +		u64 not_IP : 1;
> +		u64 rcv_error : 1;
> +		u64 err_code : 8;
> +	} snoip_cn68xx;
> +	struct {
> +		u64 bufs : 8;
> +		u64 unused : 8;
> +		u64 vlan_valid : 1;
> +		u64 vlan_stacked : 1;
> +		u64 unassigned : 1;
> +		u64 vlan_cfi : 1;
> +		u64 vlan_id : 12;
> +		u64 pr : 4;
> +		u64 unassigned2a : 8;
> +		u64 unassigned2 : 4;
> +		u64 software : 1;
> +		u64 unassigned3 : 1;
> +		u64 is_rarp : 1;
> +		u64 is_arp : 1;
> +		u64 is_bcast : 1;
> +		u64 is_mcast : 1;
> +		u64 not_IP : 1;
> +		u64 rcv_error : 1;
> +		u64 err_code : 8;
> +	} snoip_cn38xx;
> +} cvmx_pip_wqe_word2_t;
> +
> +typedef union {
> +	u64 u64;
> +	struct {
> +		u64 software : 1;
> +		u64 lg_hdr_type : 5;
> +		u64 lf_hdr_type : 5;
> +		u64 le_hdr_type : 5;
> +		u64 ld_hdr_type : 5;
> +		u64 lc_hdr_type : 5;
> +		u64 lb_hdr_type : 5;
> +		u64 is_la_ether : 1;
> +		u64 rsvd_0 : 8;
> +		u64 vlan_valid : 1;
> +		u64 vlan_stacked : 1;
> +		u64 stat_inc : 1;
> +		u64 pcam_flag4 : 1;
> +		u64 pcam_flag3 : 1;
> +		u64 pcam_flag2 : 1;
> +		u64 pcam_flag1 : 1;
> +		u64 is_frag : 1;
> +		u64 is_l3_bcast : 1;
> +		u64 is_l3_mcast : 1;
> +		u64 is_l2_bcast : 1;
> +		u64 is_l2_mcast : 1;
> +		u64 is_raw : 1;
> +		u64 err_level : 3;
> +		u64 err_code : 8;
> +	};
> +} cvmx_pki_wqe_word2_t;
> +
> +typedef union {
> +	u64 u64;
> +	cvmx_pki_wqe_word2_t pki;
> +	cvmx_pip_wqe_word2_t pip;
> +} cvmx_wqe_word2_t;
> +
> +typedef union {
> +	u64 u64;
> +	struct {
> +		u16 hw_chksum;
> +		u8 unused;
> +		u64 next_ptr : 40;
> +	} cn38xx;
> +	struct {
> +		u64 l4ptr : 8;	  /* 56..63 */
> +		u64 unused0 : 8;  /* 48..55 */
> +		u64 l3ptr : 8;	  /* 40..47 */
> +		u64 l2ptr : 8;	  /* 32..39 */
> +		u64 unused1 : 18; /* 14..31 */
> +		u64 bpid : 6;	  /* 8..13 */
> +		u64 unused2 : 2;  /* 6..7 */
> +		u64 pknd : 6;	  /* 0..5 */
> +	} cn68xx;
> +} cvmx_pip_wqe_word0_t;
> +
> +typedef union {
> +	u64 u64;
> +	struct {
> +		u64 rsvd_0 : 4;
> +		u64 aura : 12;
> +		u64 rsvd_1 : 1;
> +		u64 apad : 3;
> +		u64 channel : 12;
> +		u64 bufs : 8;
> +		u64 style : 8;
> +		u64 rsvd_2 : 10;
> +		u64 pknd : 6;
> +	};
> +} cvmx_pki_wqe_word0_t;
> +
> +/* Use reserved bit, set by HW to 0, to indicate buf_ptr legacy
> translation*/
> +#define pki_wqe_translated word0.rsvd_1
> +
> +typedef union {
> +	u64 u64;
> +	cvmx_pip_wqe_word0_t pip;
> +	cvmx_pki_wqe_word0_t pki;
> +	struct {
> +		u64 unused : 24;
> +		u64 next_ptr : 40; /* On cn68xx this is unused as well
> */
> +	} raw;
> +} cvmx_wqe_word0_t;
> +
> +typedef union {
> +	u64 u64;
> +	struct {
> +		u64 len : 16;
> +		u64 rsvd_0 : 2;
> +		u64 rsvd_1 : 2;
> +		u64 grp : 10;
> +		cvmx_pow_tag_type_t tag_type : 2;
> +		u64 tag : 32;
> +	};
> +} cvmx_pki_wqe_word1_t;
> +
> +#define pki_errata20776 word1.rsvd_0
> +
> +typedef union {
> +	u64 u64;
> +	struct {
> +		u64 len : 16;
> +		u64 varies : 14;
> +		cvmx_pow_tag_type_t tag_type : 2;
> +		u64 tag : 32;
> +	};
> +	cvmx_pki_wqe_word1_t cn78xx;
> +	struct {
> +		u64 len : 16;
> +		u64 zero_0 : 1;
> +		u64 qos : 3;
> +		u64 zero_1 : 1;
> +		u64 grp : 6;
> +		u64 zero_2 : 3;
> +		cvmx_pow_tag_type_t tag_type : 2;
> +		u64 tag : 32;
> +	} cn68xx;
> +	struct {
> +		u64 len : 16;
> +		u64 ipprt : 6;
> +		u64 qos : 3;
> +		u64 grp : 4;
> +		u64 zero_2 : 1;
> +		cvmx_pow_tag_type_t tag_type : 2;
> +		u64 tag : 32;
> +	} cn38xx;
> +} cvmx_wqe_word1_t;
> +
> +typedef union {
> +	u64 u64;
> +	struct {
> +		u64 rsvd_0 : 8;
> +		u64 hwerr : 8;
> +		u64 rsvd_1 : 24;
> +		u64 sqid : 8;
> +		u64 rsvd_2 : 4;
> +		u64 vfnum : 12;
> +	};
> +} cvmx_wqe_word3_t;
> +
> +typedef union {
> +	u64 u64;
> +	struct {
> +		u64 rsvd_0 : 21;
> +		u64 sqfc : 11;
> +		u64 rsvd_1 : 5;
> +		u64 sqtail : 11;
> +		u64 rsvd_2 : 3;
> +		u64 sqhead : 13;
> +	};
> +} cvmx_wqe_word4_t;
> +
> +/**
> + * Work queue entry format.
> + * Must be 8-byte aligned.
> + */
> +typedef struct cvmx_wqe_s {
> +	/*-------------------------------------------------------------
> ------*/
> +	/* WORD
> 0                                                            */
> +	/*-------------------------------------------------------------
> ------*/
> +	/* HW WRITE: the following 64 bits are filled by HW when a
> packet
> +	 * arrives.
> +	 */
> +	cvmx_wqe_word0_t word0;
> +
> +	/*-------------------------------------------------------------
> ------*/
> +	/* WORD
> 1                                                            */
> +	/*-------------------------------------------------------------
> ------*/
> +	/* HW WRITE: the following 64 bits are filled by HW when a
> packet
> +	 * arrives.
> +	 */
> +	cvmx_wqe_word1_t word1;
> +
> +	/*-------------------------------------------------------------
> ------*/
> +	/* WORD
> 2                                                            */
> +	/*-------------------------------------------------------------
> ------*/
> +	/* HW WRITE: the following 64-bits are filled in by hardware
> when a
> +	 * packet arrives. This indicates a variety of status and error
> +	 *conditions.
> +	 */
> +	cvmx_pip_wqe_word2_t word2;
> +
> +	/* Pointer to the first segment of the packet. */
> +	cvmx_buf_ptr_t packet_ptr;
> +
> +	/* HW WRITE: OCTEON will fill in a programmable amount from the
> packet,
> +	 * up to (at most, but perhaps less) the amount needed to fill
> the work
> +	 * queue entry to 128 bytes. If the packet is recognized to be
> IP, the
> +	 * hardware starts (except that the IPv4 header is padded for
> +	 * appropriate alignment) writing here where the IP header
> starts.
> +	 * If the packet is not recognized to be IP, the hardware
> starts
> +	 * writing the beginning of the packet here.
> +	 */
> +	u8 packet_data[96];
> +
> +	/* If desired, SW can make the work Q entry any length. For the
> purposes
> +	 * of discussion here, Assume 128B always, as this is all that
> the hardware
> +	 * deals with.
> +	 */
> +} CVMX_CACHE_LINE_ALIGNED cvmx_wqe_t;
> +
> +/**
> + * Work queue entry format for NQM
> + * Must be 8-byte aligned
> + */
> +typedef struct cvmx_wqe_nqm_s {
> +	/*-------------------------------------------------------------
> ------*/
> +	/* WORD
> 0                                                            */
> +	/*-------------------------------------------------------------
> ------*/
> +	/* HW WRITE: the following 64 bits are filled by HW when a
> packet
> +	 * arrives.
> +	 */
> +	cvmx_wqe_word0_t word0;
> +
> +	/*-------------------------------------------------------------
> ------*/
> +	/* WORD
> 1                                                            */
> +	/*-------------------------------------------------------------
> ------*/
> +	/* HW WRITE: the following 64 bits are filled by HW when a
> packet
> +	 * arrives.
> +	 */
> +	cvmx_wqe_word1_t word1;
> +
> +	/*-------------------------------------------------------------
> ------*/
> +	/* WORD
> 2                                                            */
> +	/*-------------------------------------------------------------
> ------*/
> +	/* Reserved */
> +	u64 word2;
> +
> +	/*-------------------------------------------------------------
> ------*/
> +	/* WORD
> 3                                                            */
> +	/*-------------------------------------------------------------
> ------*/
> +	/* NVMe specific information.*/
> +	cvmx_wqe_word3_t word3;
> +
> +	/*-------------------------------------------------------------
> ------*/
> +	/* WORD
> 4                                                            */
> +	/*-------------------------------------------------------------
> ------*/
> +	/* NVMe specific information.*/
> +	cvmx_wqe_word4_t word4;
> +
> +	/* HW WRITE: OCTEON will fill in a programmable amount from the
> packet,
> +	 * up to (at most, but perhaps less) the amount needed to fill
> the work
> +	 * queue entry to 128 bytes. If the packet is recognized to be
> IP, the
> +	 * hardware starts (except that the IPv4 header is padded for
> +	 * appropriate alignment) writing here where the IP header
> starts.
> +	 * If the packet is not recognized to be IP, the hardware
> starts
> +	 * writing the beginning of the packet here.
> +	 */
> +	u8 packet_data[88];
> +
> +	/* If desired, SW can make the work Q entry any length.
> +	 * For the purposes of discussion here, assume 128B always, as
> this is
> +	 * all that the hardware deals with.
> +	 */
> +} CVMX_CACHE_LINE_ALIGNED cvmx_wqe_nqm_t;
> +
> +/**
> + * Work queue entry format for 78XX.
> + * In 78XX packet data always resides in WQE buffer unless option
> + * DIS_WQ_DAT=1 in PKI_STYLE_BUF, which causes packet data to use
> separate buffer.
> + *
> + * Must be 8-byte aligned.
> + */
> +typedef struct {
> +	/*-------------------------------------------------------------
> ------*/
> +	/* WORD
> 0                                                            */
> +	/*-------------------------------------------------------------
> ------*/
> +	/* HW WRITE: the following 64 bits are filled by HW when a
> packet
> +	 * arrives.
> +	 */
> +	cvmx_pki_wqe_word0_t word0;
> +
> +	/*-------------------------------------------------------------
> ------*/
> +	/* WORD
> 1                                                            */
> +	/*-------------------------------------------------------------
> ------*/
> +	/* HW WRITE: the following 64 bits are filled by HW when a
> packet
> +	 * arrives.
> +	 */
> +	cvmx_pki_wqe_word1_t word1;
> +
> +	/*-------------------------------------------------------------
> ------*/
> +	/* WORD
> 2                                                            */
> +	/*-------------------------------------------------------------
> ------*/
> +	/* HW WRITE: the following 64-bits are filled in by hardware
> when a
> +	 * packet arrives. This indicates a variety of status and error
> +	 * conditions.
> +	 */
> +	cvmx_pki_wqe_word2_t word2;
> +
> +	/*-------------------------------------------------------------
> ------*/
> +	/* WORD
> 3                                                            */
> +	/*-------------------------------------------------------------
> ------*/
> +	/* Pointer to the first segment of the packet.*/
> +	cvmx_buf_ptr_pki_t packet_ptr;
> +
> +	/*-------------------------------------------------------------
> ------*/
> +	/* WORD
> 4                                                            */
> +	/*-------------------------------------------------------------
> ------*/
> +	/* HW WRITE: the following 64-bits are filled in by hardware
> when a
> +	 * packet arrives contains a byte pointer to the start of Layer
> +	 * A/B/C/D/E/F/G relative of start of packet.
> +	 */
> +	cvmx_pki_wqe_word4_t word4;
> +
> +	/*-------------------------------------------------------------
> ------*/
> +	/* WORDs 5/6/7 may be extended there, if WQE_HSZ is
> set.             */
> +	/*-------------------------------------------------------------
> ------*/
> +	u64 wqe_data[11];
> +
> +} CVMX_CACHE_LINE_ALIGNED cvmx_wqe_78xx_t;
> +
> +/* Node LS-bit position in the WQE[grp] or PKI_QPG_TBL[grp_ok].*/
> +#define CVMX_WQE_GRP_NODE_SHIFT 8
> +
> +/*
> + * This is an accessor function into the WQE that retreives the
> + * ingress port number, which can also be used as a destination
> + * port number for the same port.
> + *
> + * @param work - Work Queue Entrey pointer
> + * @returns returns the normalized port number, also known as "ipd"
> port
> + */
> +static inline int cvmx_wqe_get_port(cvmx_wqe_t *work)
> +{
> +	int port;
> +
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		/* In 78xx wqe entry has channel number not port*/
> +		port = work->word0.pki.channel;
> +		/* For BGX interfaces (0x800 - 0xdff) the 4 LSBs
> indicate
> +		 * the PFC channel, must be cleared to normalize to
> "ipd"
> +		 */
> +		if (port & 0x800)
> +			port &= 0xff0;
> +		/* Node number is in AURA field, make it part of port #
> */
> +		port |= (work->word0.pki.aura >> 10) << 12;
> +	} else if (octeon_has_feature(OCTEON_FEATURE_CN68XX_WQE)) {
> +		port = work->word2.s_cn68xx.port;
> +	} else {
> +		port = work->word1.cn38xx.ipprt;
> +	}
> +
> +	return port;
> +}
> +
> +static inline void cvmx_wqe_set_port(cvmx_wqe_t *work, int port)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE))
> +		work->word0.pki.channel = port;
> +	else if (octeon_has_feature(OCTEON_FEATURE_CN68XX_WQE))
> +		work->word2.s_cn68xx.port = port;
> +	else
> +		work->word1.cn38xx.ipprt = port;
> +}
> +
> +static inline int cvmx_wqe_get_grp(cvmx_wqe_t *work)
> +{
> +	int grp;
> +
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE))
> +		/* legacy: GRP[0..2] :=QOS */
> +		grp = (0xff & work->word1.cn78xx.grp) >> 3;
> +	else if (octeon_has_feature(OCTEON_FEATURE_CN68XX_WQE))
> +		grp = work->word1.cn68xx.grp;
> +	else
> +		grp = work->word1.cn38xx.grp;
> +
> +	return grp;
> +}
> +
> +static inline void cvmx_wqe_set_xgrp(cvmx_wqe_t *work, int grp)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE))
> +		work->word1.cn78xx.grp = grp;
> +	else if (octeon_has_feature(OCTEON_FEATURE_CN68XX_WQE))
> +		work->word1.cn68xx.grp = grp;
> +	else
> +		work->word1.cn38xx.grp = grp;
> +}
> +
> +static inline int cvmx_wqe_get_xgrp(cvmx_wqe_t *work)
> +{
> +	int grp;
> +
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE))
> +		grp = work->word1.cn78xx.grp;
> +	else if (octeon_has_feature(OCTEON_FEATURE_CN68XX_WQE))
> +		grp = work->word1.cn68xx.grp;
> +	else
> +		grp = work->word1.cn38xx.grp;
> +
> +	return grp;
> +}
> +
> +static inline void cvmx_wqe_set_grp(cvmx_wqe_t *work, int grp)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		unsigned int node = cvmx_get_node_num();
> +		/* Legacy: GRP[0..2] :=QOS */
> +		work->word1.cn78xx.grp &= 0x7;
> +		work->word1.cn78xx.grp |= 0xff & (grp << 3);
> +		work->word1.cn78xx.grp |= (node << 8);
> +	} else if (octeon_has_feature(OCTEON_FEATURE_CN68XX_WQE)) {
> +		work->word1.cn68xx.grp = grp;
> +	} else {
> +		work->word1.cn38xx.grp = grp;
> +	}
> +}
> +
> +static inline int cvmx_wqe_get_qos(cvmx_wqe_t *work)
> +{
> +	int qos;
> +
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		/* Legacy: GRP[0..2] :=QOS */
> +		qos = work->word1.cn78xx.grp & 0x7;
> +	} else if (octeon_has_feature(OCTEON_FEATURE_CN68XX_WQE)) {
> +		qos = work->word1.cn68xx.qos;
> +	} else {
> +		qos = work->word1.cn38xx.qos;
> +	}
> +
> +	return qos;
> +}
> +
> +static inline void cvmx_wqe_set_qos(cvmx_wqe_t *work, int qos)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		/* legacy: GRP[0..2] :=QOS */
> +		work->word1.cn78xx.grp &= ~0x7;
> +		work->word1.cn78xx.grp |= qos & 0x7;
> +	} else if (octeon_has_feature(OCTEON_FEATURE_CN68XX_WQE)) {
> +		work->word1.cn68xx.qos = qos;
> +	} else {
> +		work->word1.cn38xx.qos = qos;
> +	}
> +}
> +
> +static inline int cvmx_wqe_get_len(cvmx_wqe_t *work)
> +{
> +	int len;
> +
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE))
> +		len = work->word1.cn78xx.len;
> +	else if (octeon_has_feature(OCTEON_FEATURE_CN68XX_WQE))
> +		len = work->word1.cn68xx.len;
> +	else
> +		len = work->word1.cn38xx.len;
> +
> +	return len;
> +}
> +
> +static inline void cvmx_wqe_set_len(cvmx_wqe_t *work, int len)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE))
> +		work->word1.cn78xx.len = len;
> +	else if (octeon_has_feature(OCTEON_FEATURE_CN68XX_WQE))
> +		work->word1.cn68xx.len = len;
> +	else
> +		work->word1.cn38xx.len = len;
> +}
> +
> +/**
> + * This function returns, if there was L2/L1 errors detected in
> packet.
> + *
> + * @param work	pointer to work queue entry
> + *
> + * @return	0 if packet had no error, non-zero to indicate error
> code.
> + *
> + * Please refer to HRM for the specific model for full enumaration
> of error codes.
> + * With Octeon1/Octeon2 models, the returned code indicates L1/L2
> errors.
> + * On CN73XX/CN78XX, the return code is the value of PKI_OPCODE_E,
> + * if it is non-zero, otherwise the returned code will be derived
> from
> + * PKI_ERRLEV_E such that an error indicated in LayerA will return
> 0x20,
> + * LayerB - 0x30, LayerC - 0x40 and so forth.
> + */
> +static inline int cvmx_wqe_get_rcv_err(cvmx_wqe_t *work)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		cvmx_wqe_78xx_t *wqe = (cvmx_wqe_78xx_t *)work;
> +
> +		if (wqe->word2.err_level == CVMX_PKI_ERRLEV_E_RE ||
> wqe->word2.err_code != 0)
> +			return wqe->word2.err_code;
> +		else
> +			return (wqe->word2.err_level << 4) + 0x10;
> +	} else if (work->word2.snoip.rcv_error) {
> +		return work->word2.snoip.err_code;
> +	}
> +
> +	return 0;
> +}
> +
> +static inline u32 cvmx_wqe_get_tag(cvmx_wqe_t *work)
> +{
> +	return work->word1.tag;
> +}
> +
> +static inline void cvmx_wqe_set_tag(cvmx_wqe_t *work, u32 tag)
> +{
> +	work->word1.tag = tag;
> +}
> +
> +static inline int cvmx_wqe_get_tt(cvmx_wqe_t *work)
> +{
> +	return work->word1.tag_type;
> +}
> +
> +static inline void cvmx_wqe_set_tt(cvmx_wqe_t *work, int tt)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		work->word1.cn78xx.tag_type = (cvmx_pow_tag_type_t)tt;
> +	} else if (octeon_has_feature(OCTEON_FEATURE_CN68XX_WQE)) {
> +		work->word1.cn68xx.tag_type = (cvmx_pow_tag_type_t)tt;
> +		work->word1.cn68xx.zero_2 = 0;
> +	} else {
> +		work->word1.cn38xx.tag_type = (cvmx_pow_tag_type_t)tt;
> +		work->word1.cn38xx.zero_2 = 0;
> +	}
> +}
> +
> +static inline u8 cvmx_wqe_get_unused8(cvmx_wqe_t *work)
> +{
> +	u8 bits;
> +
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		cvmx_wqe_78xx_t *wqe = (cvmx_wqe_78xx_t *)work;
> +
> +		bits = wqe->word2.rsvd_0;
> +	} else if (octeon_has_feature(OCTEON_FEATURE_CN68XX_WQE)) {
> +		bits = work->word0.pip.cn68xx.unused1;
> +	} else {
> +		bits = work->word0.pip.cn38xx.unused;
> +	}
> +
> +	return bits;
> +}
> +
> +static inline void cvmx_wqe_set_unused8(cvmx_wqe_t *work, u8 v)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		cvmx_wqe_78xx_t *wqe = (cvmx_wqe_78xx_t *)work;
> +
> +		wqe->word2.rsvd_0 = v;
> +	} else if (octeon_has_feature(OCTEON_FEATURE_CN68XX_WQE)) {
> +		work->word0.pip.cn68xx.unused1 = v;
> +	} else {
> +		work->word0.pip.cn38xx.unused = v;
> +	}
> +}
> +
> +static inline u8 cvmx_wqe_get_user_flags(cvmx_wqe_t *work)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE))
> +		return work->word0.pki.rsvd_2;
> +	else
> +		return 0;
> +}
> +
> +static inline void cvmx_wqe_set_user_flags(cvmx_wqe_t *work, u8 v)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE))
> +		work->word0.pki.rsvd_2 = v;
> +}
> +
> +static inline int cvmx_wqe_get_channel(cvmx_wqe_t *work)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE))
> +		return (work->word0.pki.channel);
> +	else
> +		return cvmx_wqe_get_port(work);
> +}
> +
> +static inline void cvmx_wqe_set_channel(cvmx_wqe_t *work, int
> channel)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE))
> +		work->word0.pki.channel = channel;
> +	else
> +		debug("%s: ERROR: not supported for model\n",
> __func__);
> +}
> +
> +static inline int cvmx_wqe_get_aura(cvmx_wqe_t *work)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE))
> +		return (work->word0.pki.aura);
> +	else
> +		return (work->packet_ptr.s.pool);
> +}
> +
> +static inline void cvmx_wqe_set_aura(cvmx_wqe_t *work, int aura)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE))
> +		work->word0.pki.aura = aura;
> +	else
> +		work->packet_ptr.s.pool = aura;
> +}
> +
> +static inline int cvmx_wqe_get_style(cvmx_wqe_t *work)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE))
> +		return (work->word0.pki.style);
> +	return 0;
> +}
> +
> +static inline void cvmx_wqe_set_style(cvmx_wqe_t *work, int style)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE))
> +		work->word0.pki.style = style;
> +}
> +
> +static inline int cvmx_wqe_is_l3_ip(cvmx_wqe_t *work)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		cvmx_wqe_78xx_t *wqe = (cvmx_wqe_78xx_t *)work;
> +		/* Match all 4 values for v4/v6 with.without options */
> +		if ((wqe->word2.lc_hdr_type & 0x1c) ==
> CVMX_PKI_LTYPE_E_IP4)
> +			return 1;
> +		if ((wqe->word2.le_hdr_type & 0x1c) ==
> CVMX_PKI_LTYPE_E_IP4)
> +			return 1;
> +		return 0;
> +	} else {
> +		return !work->word2.s_cn38xx.not_IP;
> +	}
> +}
> +
> +static inline int cvmx_wqe_is_l3_ipv4(cvmx_wqe_t *work)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		cvmx_wqe_78xx_t *wqe = (cvmx_wqe_78xx_t *)work;
> +		/* Match 2 values - with/wotuout options */
> +		if ((wqe->word2.lc_hdr_type & 0x1e) ==
> CVMX_PKI_LTYPE_E_IP4)
> +			return 1;
> +		if ((wqe->word2.le_hdr_type & 0x1e) ==
> CVMX_PKI_LTYPE_E_IP4)
> +			return 1;
> +		return 0;
> +	} else {
> +		return (!work->word2.s_cn38xx.not_IP &&
> +			!work->word2.s_cn38xx.is_v6);
> +	}
> +}
> +
> +static inline int cvmx_wqe_is_l3_ipv6(cvmx_wqe_t *work)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		cvmx_wqe_78xx_t *wqe = (cvmx_wqe_78xx_t *)work;
> +		/* Match 2 values - with/wotuout options */
> +		if ((wqe->word2.lc_hdr_type & 0x1e) ==
> CVMX_PKI_LTYPE_E_IP6)
> +			return 1;
> +		if ((wqe->word2.le_hdr_type & 0x1e) ==
> CVMX_PKI_LTYPE_E_IP6)
> +			return 1;
> +		return 0;
> +	} else {
> +		return (!work->word2.s_cn38xx.not_IP &&
> +			work->word2.s_cn38xx.is_v6);
> +	}
> +}
> +
> +static inline bool cvmx_wqe_is_l4_udp_or_tcp(cvmx_wqe_t *work)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		cvmx_wqe_78xx_t *wqe = (cvmx_wqe_78xx_t *)work;
> +
> +		if (wqe->word2.lf_hdr_type == CVMX_PKI_LTYPE_E_TCP)
> +			return true;
> +		if (wqe->word2.lf_hdr_type == CVMX_PKI_LTYPE_E_UDP)
> +			return true;
> +		return false;
> +	}
> +
> +	if (work->word2.s_cn38xx.not_IP)
> +		return false;
> +
> +	return (work->word2.s_cn38xx.tcp_or_udp != 0);
> +}
> +
> +static inline int cvmx_wqe_is_l2_bcast(cvmx_wqe_t *work)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		cvmx_wqe_78xx_t *wqe = (cvmx_wqe_78xx_t *)work;
> +
> +		return wqe->word2.is_l2_bcast;
> +	} else {
> +		return work->word2.s_cn38xx.is_bcast;
> +	}
> +}
> +
> +static inline int cvmx_wqe_is_l2_mcast(cvmx_wqe_t *work)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		cvmx_wqe_78xx_t *wqe = (cvmx_wqe_78xx_t *)work;
> +
> +		return wqe->word2.is_l2_mcast;
> +	} else {
> +		return work->word2.s_cn38xx.is_mcast;
> +	}
> +}
> +
> +static inline void cvmx_wqe_set_l2_bcast(cvmx_wqe_t *work, bool
> bcast)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		cvmx_wqe_78xx_t *wqe = (cvmx_wqe_78xx_t *)work;
> +
> +		wqe->word2.is_l2_bcast = bcast;
> +	} else {
> +		work->word2.s_cn38xx.is_bcast = bcast;
> +	}
> +}
> +
> +static inline void cvmx_wqe_set_l2_mcast(cvmx_wqe_t *work, bool
> mcast)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		cvmx_wqe_78xx_t *wqe = (cvmx_wqe_78xx_t *)work;
> +
> +		wqe->word2.is_l2_mcast = mcast;
> +	} else {
> +		work->word2.s_cn38xx.is_mcast = mcast;
> +	}
> +}
> +
> +static inline int cvmx_wqe_is_l3_bcast(cvmx_wqe_t *work)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		cvmx_wqe_78xx_t *wqe = (cvmx_wqe_78xx_t *)work;
> +
> +		return wqe->word2.is_l3_bcast;
> +	}
> +	debug("%s: ERROR: not supported for model\n", __func__);
> +	return 0;
> +}
> +
> +static inline int cvmx_wqe_is_l3_mcast(cvmx_wqe_t *work)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		cvmx_wqe_78xx_t *wqe = (cvmx_wqe_78xx_t *)work;
> +
> +		return wqe->word2.is_l3_mcast;
> +	}
> +	debug("%s: ERROR: not supported for model\n", __func__);
> +	return 0;
> +}
> +
> +/**
> + * This function returns is there was IP error detected in packet.
> + * For 78XX it does not flag ipv4 options and ipv6 extensions.
> + * For older chips if PIP_GBL_CTL was proviosned to flag ip4_otions
> and
> + * ipv6 extension, it will be flag them.
> + * @param work	pointer to work queue entry
> + * @return	1 -- If IP error was found in packet
> + *          0 -- If no IP error was found in packet.
> + */
> +static inline int cvmx_wqe_is_ip_exception(cvmx_wqe_t *work)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		cvmx_wqe_78xx_t *wqe = (cvmx_wqe_78xx_t *)work;
> +
> +		if (wqe->word2.err_level == CVMX_PKI_ERRLEV_E_LC)
> +			return 1;
> +		else
> +			return 0;
> +	}
> +
> +	return work->word2.s.IP_exc;
> +}
> +
> +static inline int cvmx_wqe_is_l4_error(cvmx_wqe_t *work)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		cvmx_wqe_78xx_t *wqe = (cvmx_wqe_78xx_t *)work;
> +
> +		if (wqe->word2.err_level == CVMX_PKI_ERRLEV_E_LF)
> +			return 1;
> +		else
> +			return 0;
> +	} else {
> +		return work->word2.s.L4_error;
> +	}
> +}
> +
> +static inline void cvmx_wqe_set_vlan(cvmx_wqe_t *work, bool set)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		cvmx_wqe_78xx_t *wqe = (cvmx_wqe_78xx_t *)work;
> +
> +		wqe->word2.vlan_valid = set;
> +	} else {
> +		work->word2.s.vlan_valid = set;
> +	}
> +}
> +
> +static inline int cvmx_wqe_is_vlan(cvmx_wqe_t *work)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		cvmx_wqe_78xx_t *wqe = (cvmx_wqe_78xx_t *)work;
> +
> +		return wqe->word2.vlan_valid;
> +	} else {
> +		return work->word2.s.vlan_valid;
> +	}
> +}
> +
> +static inline int cvmx_wqe_is_vlan_stacked(cvmx_wqe_t *work)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		cvmx_wqe_78xx_t *wqe = (cvmx_wqe_78xx_t *)work;
> +
> +		return wqe->word2.vlan_stacked;
> +	} else {
> +		return work->word2.s.vlan_stacked;
> +	}
> +}
> +
> +/**
> + * Extract packet data buffer pointer from work queue entry.
> + *
> + * Returns the legacy (Octeon1/Octeon2) buffer pointer structure
> + * for the linked buffer list.
> + * On CN78XX, the native buffer pointer structure is converted into
> + * the legacy format.
> + * The legacy buf_ptr is then stored in the WQE, and word0 reserved
> + * field is set to indicate that the buffer pointers were
> translated.
> + * If the packet data is only found inside the work queue entry,
> + * a standard buffer pointer structure is created for it.
> + */
> +cvmx_buf_ptr_t cvmx_wqe_get_packet_ptr(cvmx_wqe_t *work);
> +
> +static inline int cvmx_wqe_get_bufs(cvmx_wqe_t *work)
> +{
> +	int bufs;
> +
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		bufs = work->word0.pki.bufs;
> +	} else {
> +		/* Adjust for packet-in-WQE cases */
> +		if (cvmx_unlikely(work->word2.s_cn38xx.bufs == 0 &&
> !work->word2.s.software))
> +			(void)cvmx_wqe_get_packet_ptr(work);
> +		bufs = work->word2.s_cn38xx.bufs;
> +	}
> +	return bufs;
> +}
> +
> +/**
> + * Free Work Queue Entry memory
> + *
> + * Will return the WQE buffer to its pool, unless the WQE contains
> + * non-redundant packet data.
> + * This function is intended to be called AFTER the packet data
> + * has been passed along to PKO for transmission and release.
> + * It can also follow a call to cvmx_helper_free_packet_data()
> + * to release the WQE after associated data was released.
> + */
> +void cvmx_wqe_free(cvmx_wqe_t *work);
> +
> +/**
> + * Check if a work entry has been intiated by software
> + *
> + */
> +static inline bool cvmx_wqe_is_soft(cvmx_wqe_t *work)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		cvmx_wqe_78xx_t *wqe = (cvmx_wqe_78xx_t *)work;
> +
> +		return wqe->word2.software;
> +	} else {
> +		return work->word2.s.software;
> +	}
> +}
> +
> +/**
> + * Allocate a work-queue entry for delivering software-initiated
> + * event notifications.
> + * The application data is copied into the work-queue entry,
> + * if the space is sufficient.
> + */
> +cvmx_wqe_t *cvmx_wqe_soft_create(void *data_p, unsigned int
> data_sz);
> +
> +/* Errata (PKI-20776) PKI_BUFLINK_S's are endian-swapped
> + * CN78XX pass 1.x has a bug where the packet pointer in each
> segment is
> + * written in the opposite endianness of the configured mode. Fix
> these here.
> + */
> +static inline void cvmx_wqe_pki_errata_20776(cvmx_wqe_t *work)
> +{
> +	cvmx_wqe_78xx_t *wqe = (cvmx_wqe_78xx_t *)work;
> +
> +	if (OCTEON_IS_MODEL(OCTEON_CN78XX_PASS1_X) && !wqe-
> >pki_errata20776) {
> +		u64 bufs;
> +		cvmx_buf_ptr_pki_t buffer_next;
> +
> +		bufs = wqe->word0.bufs;
> +		buffer_next = wqe->packet_ptr;
> +		while (bufs > 1) {
> +			cvmx_buf_ptr_pki_t next;
> +			void *nextaddr =
> cvmx_phys_to_ptr(buffer_next.addr - 8);
> +
> +			memcpy(&next, nextaddr, sizeof(next));
> +			next.u64 = __builtin_bswap64(next.u64);
> +			memcpy(nextaddr, &next, sizeof(next));
> +			buffer_next = next;
> +			bufs--;
> +		}
> +		wqe->pki_errata20776 = 1;
> +	}
> +}
> +
> +/**
> + * @INTERNAL
> + *
> + * Extract the native PKI-specific buffer pointer from WQE.
> + *
> + * NOTE: Provisional, may be superceded.
> + */
> +static inline cvmx_buf_ptr_pki_t cvmx_wqe_get_pki_pkt_ptr(cvmx_wqe_t
> *work)
> +{
> +	cvmx_wqe_78xx_t *wqe = (cvmx_wqe_78xx_t *)work;
> +
> +	if (!octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		cvmx_buf_ptr_pki_t x = { 0 };
> +		return x;
> +	}
> +
> +	cvmx_wqe_pki_errata_20776(work);
> +	return wqe->packet_ptr;
> +}
> +
> +/**
> + * Set the buffer segment count for a packet.
> + *
> + * @return Returns the actual resulting value in the WQE fielda
> + *
> + */
> +static inline unsigned int cvmx_wqe_set_bufs(cvmx_wqe_t *work,
> unsigned int bufs)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		work->word0.pki.bufs = bufs;
> +		return work->word0.pki.bufs;
> +	}
> +
> +	work->word2.s.bufs = bufs;
> +	return work->word2.s.bufs;
> +}
> +
> +/**
> + * Get the offset of Layer-3 header,
> + * only supported when Layer-3 protocol is IPv4 or IPv6.
> + *
> + * @return Returns the offset, or 0 if the offset is not known or
> unsupported.
> + *
> + * FIXME: Assuming word4 is present.
> + */
> +static inline unsigned int cvmx_wqe_get_l3_offset(cvmx_wqe_t *work)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		cvmx_wqe_78xx_t *wqe = (cvmx_wqe_78xx_t *)work;
> +		/* Match 4 values: IPv4/v6 w/wo options */
> +		if ((wqe->word2.lc_hdr_type & 0x1c) ==
> CVMX_PKI_LTYPE_E_IP4)
> +			return wqe->word4.ptr_layer_c;
> +	} else {
> +		return work->word2.s.ip_offset;
> +	}
> +
> +	return 0;
> +}
> +
> +/**
> + * Set the offset of Layer-3 header in a packet.
> + * Typically used when an IP packet is generated by software
> + * or when the Layer-2 header length is modified, and
> + * a subsequent recalculation of checksums is anticipated.
> + *
> + * @return Returns the actual value of the work entry offset field.
> + *
> + * FIXME: Assuming word4 is present.
> + */
> +static inline unsigned int cvmx_wqe_set_l3_offset(cvmx_wqe_t *work,
> unsigned int ip_off)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		cvmx_wqe_78xx_t *wqe = (cvmx_wqe_78xx_t *)work;
> +		/* Match 4 values: IPv4/v6 w/wo options */
> +		if ((wqe->word2.lc_hdr_type & 0x1c) ==
> CVMX_PKI_LTYPE_E_IP4)
> +			wqe->word4.ptr_layer_c = ip_off;
> +	} else {
> +		work->word2.s.ip_offset = ip_off;
> +	}
> +
> +	return cvmx_wqe_get_l3_offset(work);
> +}
> +
> +/**
> + * Set the indication that the packet contains a IPv4 Layer-3 *
> header.
> + * Use 'cvmx_wqe_set_l3_ipv6()' if the protocol is IPv6.
> + * When 'set' is false, the call will result in an indication
> + * that the Layer-3 protocol is neither IPv4 nor IPv6.
> + *
> + * FIXME: Add IPV4_OPT handling based on L3 header length.
> + */
> +static inline void cvmx_wqe_set_l3_ipv4(cvmx_wqe_t *work, bool set)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		cvmx_wqe_78xx_t *wqe = (cvmx_wqe_78xx_t *)work;
> +
> +		if (set)
> +			wqe->word2.lc_hdr_type = CVMX_PKI_LTYPE_E_IP4;
> +		else
> +			wqe->word2.lc_hdr_type = CVMX_PKI_LTYPE_E_NONE;
> +	} else {
> +		work->word2.s.not_IP = !set;
> +		if (set)
> +			work->word2.s_cn38xx.is_v6 = 0;
> +	}
> +}
> +
> +/**
> + * Set packet Layer-3 protocol to IPv6.
> + *
> + * FIXME: Add IPV6_OPT handling based on presence of extended
> headers.
> + */
> +static inline void cvmx_wqe_set_l3_ipv6(cvmx_wqe_t *work, bool set)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		cvmx_wqe_78xx_t *wqe = (cvmx_wqe_78xx_t *)work;
> +
> +		if (set)
> +			wqe->word2.lc_hdr_type = CVMX_PKI_LTYPE_E_IP6;
> +		else
> +			wqe->word2.lc_hdr_type = CVMX_PKI_LTYPE_E_NONE;
> +	} else {
> +		work->word2.s_cn38xx.not_IP = !set;
> +		if (set)
> +			work->word2.s_cn38xx.is_v6 = 1;
> +	}
> +}
> +
> +/**
> + * Set a packet Layer-4 protocol type to UDP.
> + */
> +static inline void cvmx_wqe_set_l4_udp(cvmx_wqe_t *work, bool set)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		cvmx_wqe_78xx_t *wqe = (cvmx_wqe_78xx_t *)work;
> +
> +		if (set)
> +			wqe->word2.lf_hdr_type = CVMX_PKI_LTYPE_E_UDP;
> +		else
> +			wqe->word2.lf_hdr_type = CVMX_PKI_LTYPE_E_NONE;
> +	} else {
> +		if (!work->word2.s_cn38xx.not_IP)
> +			work->word2.s_cn38xx.tcp_or_udp = set;
> +	}
> +}
> +
> +/**
> + * Set a packet Layer-4 protocol type to TCP.
> + */
> +static inline void cvmx_wqe_set_l4_tcp(cvmx_wqe_t *work, bool set)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		cvmx_wqe_78xx_t *wqe = (cvmx_wqe_78xx_t *)work;
> +
> +		if (set)
> +			wqe->word2.lf_hdr_type = CVMX_PKI_LTYPE_E_TCP;
> +		else
> +			wqe->word2.lf_hdr_type = CVMX_PKI_LTYPE_E_NONE;
> +	} else {
> +		if (!work->word2.s_cn38xx.not_IP)
> +			work->word2.s_cn38xx.tcp_or_udp = set;
> +	}
> +}
> +
> +/**
> + * Set the "software" flag in a work entry.
> + */
> +static inline void cvmx_wqe_set_soft(cvmx_wqe_t *work, bool set)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		cvmx_wqe_78xx_t *wqe = (cvmx_wqe_78xx_t *)work;
> +
> +		wqe->word2.software = set;
> +	} else {
> +		work->word2.s.software = set;
> +	}
> +}
> +
> +/**
> + * Return true if the packet is an IP fragment.
> + */
> +static inline bool cvmx_wqe_is_l3_frag(cvmx_wqe_t *work)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		cvmx_wqe_78xx_t *wqe = (cvmx_wqe_78xx_t *)work;
> +
> +		return (wqe->word2.is_frag != 0);
> +	}
> +
> +	if (!work->word2.s_cn38xx.not_IP)
> +		return (work->word2.s.is_frag != 0);
> +
> +	return false;
> +}
> +
> +/**
> + * Set the indicator that the packet is an fragmented IP packet.
> + */
> +static inline void cvmx_wqe_set_l3_frag(cvmx_wqe_t *work, bool set)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		cvmx_wqe_78xx_t *wqe = (cvmx_wqe_78xx_t *)work;
> +
> +		wqe->word2.is_frag = set;
> +	} else {
> +		if (!work->word2.s_cn38xx.not_IP)
> +			work->word2.s.is_frag = set;
> +	}
> +}
> +
> +/**
> + * Set the packet Layer-3 protocol to RARP.
> + */
> +static inline void cvmx_wqe_set_l3_rarp(cvmx_wqe_t *work, bool set)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		cvmx_wqe_78xx_t *wqe = (cvmx_wqe_78xx_t *)work;
> +
> +		if (set)
> +			wqe->word2.lc_hdr_type = CVMX_PKI_LTYPE_E_RARP;
> +		else
> +			wqe->word2.lc_hdr_type = CVMX_PKI_LTYPE_E_NONE;
> +	} else {
> +		work->word2.snoip.is_rarp = set;
> +	}
> +}
> +
> +/**
> + * Set the packet Layer-3 protocol to ARP.
> + */
> +static inline void cvmx_wqe_set_l3_arp(cvmx_wqe_t *work, bool set)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		cvmx_wqe_78xx_t *wqe = (cvmx_wqe_78xx_t *)work;
> +
> +		if (set)
> +			wqe->word2.lc_hdr_type = CVMX_PKI_LTYPE_E_ARP;
> +		else
> +			wqe->word2.lc_hdr_type = CVMX_PKI_LTYPE_E_NONE;
> +	} else {
> +		work->word2.snoip.is_arp = set;
> +	}
> +}
> +
> +/**
> + * Return true if the packet Layer-3 protocol is ARP.
> + */
> +static inline bool cvmx_wqe_is_l3_arp(cvmx_wqe_t *work)
> +{
> +	if (octeon_has_feature(OCTEON_FEATURE_CN78XX_WQE)) {
> +		cvmx_wqe_78xx_t *wqe = (cvmx_wqe_78xx_t *)work;
> +
> +		return (wqe->word2.lc_hdr_type ==
> CVMX_PKI_LTYPE_E_ARP);
> +	}
> +
> +	if (work->word2.s_cn38xx.not_IP)
> +		return (work->word2.snoip.is_arp != 0);
> +
> +	return false;
> +}
> +
> +#endif /* __CVMX_WQE_H__ */
> diff --git a/arch/mips/mach-octeon/include/mach/octeon_qlm.h
> b/arch/mips/mach-octeon/include/mach/octeon_qlm.h
> new file mode 100644
> index 000000000000..219625b25688
> --- /dev/null
> +++ b/arch/mips/mach-octeon/include/mach/octeon_qlm.h
> @@ -0,0 +1,109 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +/*
> + * Copyright (C) 2020 Marvell International Ltd.
> + */
> +
> +#ifndef __OCTEON_QLM_H__
> +#define __OCTEON_QLM_H__
> +
> +/* Reference clock selector values for ref_clk_sel */
> +#define OCTEON_QLM_REF_CLK_100MHZ 0 /** 100 MHz */
> +#define OCTEON_QLM_REF_CLK_125MHZ 1 /** 125 MHz */
> +#define OCTEON_QLM_REF_CLK_156MHZ 2 /** 156.25 MHz */
> +#define OCTEON_QLM_REF_CLK_161MHZ 3 /** 161.1328125 MHz */
> +
> +/**
> + * Configure qlm/dlm speed and mode.
> + * @param qlm     The QLM or DLM to configure
> + * @param speed   The speed the QLM needs to be configured in Mhz.
> + * @param mode    The QLM to be configured as SGMII/XAUI/PCIe.
> + * @param rc      Only used for PCIe, rc = 1 for root complex mode,
> 0 for EP
> + *		  mode.
> + * @param pcie_mode Only used when qlm/dlm are in pcie mode.
> + * @param ref_clk_sel Reference clock to use for 70XX where:
> + *			0: 100MHz
> + *			1: 125MHz
> + *			2: 156.25MHz
> + *			3: 161.1328125MHz (CN73XX and CN78XX only)
> + * @param ref_clk_input	This selects which reference clock
> input to use.  For
> + *			cn70xx:
> + *				0: DLMC_REF_CLK0
> + *				1: DLMC_REF_CLK1
> + *				2: DLM0_REF_CLK
> + *			cn61xx: (not used)
> + *			cn78xx/cn76xx/cn73xx:
> + *				0: Internal clock (QLM[0-7]_REF_CLK)
> + *				1: QLMC_REF_CLK0
> + *				2: QLMC_REF_CLK1
> + *
> + * @return	Return 0 on success or -1.
> + *
> + * @note	When the 161MHz clock is used it can only be used for
> + *		XLAUI mode with a 6316 speed or XFI mode with a 103125
> speed.
> + *		This rate is also only supported for CN73XX and CN78XX.
> + */
> +int octeon_configure_qlm(int qlm, int speed, int mode, int rc, int
> pcie_mode, int ref_clk_sel,
> +			 int ref_clk_input);
> +
> +int octeon_configure_qlm_cn78xx(int node, int qlm, int speed, int
> mode, int rc, int pcie_mode,
> +				int ref_clk_sel, int ref_clk_input);
> +
> +/**
> + * Some QLM speeds need to override the default tuning parameters
> + *
> + * @param node     Node to configure
> + * @param qlm      QLM to configure
> + * @param baud_mhz Desired speed in MHz
> + * @param lane     Lane the apply the tuning parameters
> + * @param tx_swing Voltage swing.  The higher the value the lower
> the voltage,
> + *		   the default value is 7.
> + * @param tx_pre   pre-cursor pre-emphasis
> + * @param tx_post  post-cursor pre-emphasis.
> + * @param tx_gain   Transmit gain. Range 0-7
> + * @param tx_vboost Transmit voltage boost. Range 0-1
> + */
> +void octeon_qlm_tune_per_lane_v3(int node, int qlm, int baud_mhz,
> int lane, int tx_swing,
> +				 int tx_pre, int tx_post, int tx_gain,
> int tx_vboost);
> +
> +/**
> + * Some QLM speeds need to override the default tuning parameters
> + *
> + * @param node     Node to configure
> + * @param qlm      QLM to configure
> + * @param baud_mhz Desired speed in MHz
> + * @param tx_swing Voltage swing.  The higher the value the lower
> the voltage,
> + *		   the default value is 7.
> + * @param tx_premptap bits [0:3] pre-cursor pre-emphasis, bits[4:8]
> post-cursor
> + *		      pre-emphasis.
> + * @param tx_gain   Transmit gain. Range 0-7
> + * @param tx_vboost Transmit voltage boost. Range 0-1
> + */
> +void octeon_qlm_tune_v3(int node, int qlm, int baud_mhz, int
> tx_swing, int tx_premptap, int tx_gain,
> +			int tx_vboost);
> +
> +/**
> + * Disables DFE for the specified QLM lane(s).
> + * This function should only be called for low-loss channels.
> + *
> + * @param node     Node to configure
> + * @param qlm      QLM to configure
> + * @param lane     Lane to configure, or -1 all lanes
> + * @param baud_mhz The speed the QLM needs to be configured in Mhz.
> + * @param mode     The QLM to be configured as SGMII/XAUI/PCIe.
> + */
> +void octeon_qlm_dfe_disable(int node, int qlm, int lane, int
> baud_mhz, int mode);
> +
> +/**
> + * Some QLMs need to override the default pre-ctle for low loss
> channels.
> + *
> + * @param node     Node to configure
> + * @param qlm      QLM to configure
> + * @param pre_ctle pre-ctle settings for low loss channels
> + */
> +void octeon_qlm_set_channel_v3(int node, int qlm, int pre_ctle);
> +
> +void octeon_init_qlm(int node);
> +
> +int octeon_mcu_probe(int node);
> +
> +#endif /* __OCTEON_QLM_H__ */
-- 
- Daniel



More information about the U-Boot mailing list