255 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Copyright (C) 2017 Marvell4 *5 * Antoine Tenart <antoine.tenart@free-electrons.com>6 */7 8#include <linux/dma-mapping.h>9#include <linux/spinlock.h>10 11#include "safexcel.h"12 13int safexcel_init_ring_descriptors(struct safexcel_crypto_priv *priv,14 struct safexcel_desc_ring *cdr,15 struct safexcel_desc_ring *rdr)16{17 int i;18 struct safexcel_command_desc *cdesc;19 dma_addr_t atok;20 21 /* Actual command descriptor ring */22 cdr->offset = priv->config.cd_offset;23 cdr->base = dmam_alloc_coherent(priv->dev,24 cdr->offset * EIP197_DEFAULT_RING_SIZE,25 &cdr->base_dma, GFP_KERNEL);26 if (!cdr->base)27 return -ENOMEM;28 cdr->write = cdr->base;29 cdr->base_end = cdr->base + cdr->offset * (EIP197_DEFAULT_RING_SIZE - 1);30 cdr->read = cdr->base;31 32 /* Command descriptor shadow ring for storing additional token data */33 cdr->shoffset = priv->config.cdsh_offset;34 cdr->shbase = dmam_alloc_coherent(priv->dev,35 cdr->shoffset *36 EIP197_DEFAULT_RING_SIZE,37 &cdr->shbase_dma, GFP_KERNEL);38 if (!cdr->shbase)39 return -ENOMEM;40 cdr->shwrite = cdr->shbase;41 cdr->shbase_end = cdr->shbase + cdr->shoffset *42 (EIP197_DEFAULT_RING_SIZE - 1);43 44 /*45 * Populate command descriptors with physical pointers to shadow descs.46 * Note that we only need to do this once if we don't overwrite them.47 */48 cdesc = cdr->base;49 atok = cdr->shbase_dma;50 for (i = 0; i < EIP197_DEFAULT_RING_SIZE; i++) {51 cdesc->atok_lo = lower_32_bits(atok);52 cdesc->atok_hi = upper_32_bits(atok);53 cdesc = (void *)cdesc + cdr->offset;54 atok += cdr->shoffset;55 }56 57 rdr->offset = priv->config.rd_offset;58 /* Use shoffset for result token offset here */59 rdr->shoffset = priv->config.res_offset;60 rdr->base = dmam_alloc_coherent(priv->dev,61 rdr->offset * EIP197_DEFAULT_RING_SIZE,62 &rdr->base_dma, GFP_KERNEL);63 if (!rdr->base)64 return -ENOMEM;65 rdr->write = rdr->base;66 rdr->base_end = rdr->base + rdr->offset * (EIP197_DEFAULT_RING_SIZE - 1);67 rdr->read = rdr->base;68 69 return 0;70}71 72inline int safexcel_select_ring(struct safexcel_crypto_priv *priv)73{74 return (atomic_inc_return(&priv->ring_used) % priv->config.rings);75}76 77static void *safexcel_ring_next_cwptr(struct safexcel_crypto_priv *priv,78 struct safexcel_desc_ring *ring,79 bool first,80 struct safexcel_token **atoken)81{82 void *ptr = ring->write;83 84 if (first)85 *atoken = ring->shwrite;86 87 if ((ring->write == ring->read - ring->offset) ||88 (ring->read == ring->base && ring->write == ring->base_end))89 return ERR_PTR(-ENOMEM);90 91 if (ring->write == ring->base_end) {92 ring->write = ring->base;93 ring->shwrite = ring->shbase;94 } else {95 ring->write += ring->offset;96 ring->shwrite += ring->shoffset;97 }98 99 return ptr;100}101 102static void *safexcel_ring_next_rwptr(struct safexcel_crypto_priv *priv,103 struct safexcel_desc_ring *ring,104 struct result_data_desc **rtoken)105{106 void *ptr = ring->write;107 108 /* Result token at relative offset shoffset */109 *rtoken = ring->write + ring->shoffset;110 111 if ((ring->write == ring->read - ring->offset) ||112 (ring->read == ring->base && ring->write == ring->base_end))113 return ERR_PTR(-ENOMEM);114 115 if (ring->write == ring->base_end)116 ring->write = ring->base;117 else118 ring->write += ring->offset;119 120 return ptr;121}122 123void *safexcel_ring_next_rptr(struct safexcel_crypto_priv *priv,124 struct safexcel_desc_ring *ring)125{126 void *ptr = ring->read;127 128 if (ring->write == ring->read)129 return ERR_PTR(-ENOENT);130 131 if (ring->read == ring->base_end)132 ring->read = ring->base;133 else134 ring->read += ring->offset;135 136 return ptr;137}138 139inline void *safexcel_ring_curr_rptr(struct safexcel_crypto_priv *priv,140 int ring)141{142 struct safexcel_desc_ring *rdr = &priv->ring[ring].rdr;143 144 return rdr->read;145}146 147inline int safexcel_ring_first_rdr_index(struct safexcel_crypto_priv *priv,148 int ring)149{150 struct safexcel_desc_ring *rdr = &priv->ring[ring].rdr;151 152 return (rdr->read - rdr->base) / rdr->offset;153}154 155inline int safexcel_ring_rdr_rdesc_index(struct safexcel_crypto_priv *priv,156 int ring,157 struct safexcel_result_desc *rdesc)158{159 struct safexcel_desc_ring *rdr = &priv->ring[ring].rdr;160 161 return ((void *)rdesc - rdr->base) / rdr->offset;162}163 164void safexcel_ring_rollback_wptr(struct safexcel_crypto_priv *priv,165 struct safexcel_desc_ring *ring)166{167 if (ring->write == ring->read)168 return;169 170 if (ring->write == ring->base) {171 ring->write = ring->base_end;172 ring->shwrite = ring->shbase_end;173 } else {174 ring->write -= ring->offset;175 ring->shwrite -= ring->shoffset;176 }177}178 179struct safexcel_command_desc *safexcel_add_cdesc(struct safexcel_crypto_priv *priv,180 int ring_id,181 bool first, bool last,182 dma_addr_t data, u32 data_len,183 u32 full_data_len,184 dma_addr_t context,185 struct safexcel_token **atoken)186{187 struct safexcel_command_desc *cdesc;188 189 cdesc = safexcel_ring_next_cwptr(priv, &priv->ring[ring_id].cdr,190 first, atoken);191 if (IS_ERR(cdesc))192 return cdesc;193 194 cdesc->particle_size = data_len;195 cdesc->rsvd0 = 0;196 cdesc->last_seg = last;197 cdesc->first_seg = first;198 cdesc->additional_cdata_size = 0;199 cdesc->rsvd1 = 0;200 cdesc->data_lo = lower_32_bits(data);201 cdesc->data_hi = upper_32_bits(data);202 203 if (first) {204 /*205 * Note that the length here MUST be >0 or else the EIP(1)97206 * may hang. Newer EIP197 firmware actually incorporates this207 * fix already, but that doesn't help the EIP97 and we may208 * also be running older firmware.209 */210 cdesc->control_data.packet_length = full_data_len ?: 1;211 cdesc->control_data.options = EIP197_OPTION_MAGIC_VALUE |212 EIP197_OPTION_64BIT_CTX |213 EIP197_OPTION_CTX_CTRL_IN_CMD |214 EIP197_OPTION_RC_AUTO;215 cdesc->control_data.type = EIP197_TYPE_BCLA;216 cdesc->control_data.context_lo = lower_32_bits(context) |217 EIP197_CONTEXT_SMALL;218 cdesc->control_data.context_hi = upper_32_bits(context);219 }220 221 return cdesc;222}223 224struct safexcel_result_desc *safexcel_add_rdesc(struct safexcel_crypto_priv *priv,225 int ring_id,226 bool first, bool last,227 dma_addr_t data, u32 len)228{229 struct safexcel_result_desc *rdesc;230 struct result_data_desc *rtoken;231 232 rdesc = safexcel_ring_next_rwptr(priv, &priv->ring[ring_id].rdr,233 &rtoken);234 if (IS_ERR(rdesc))235 return rdesc;236 237 rdesc->particle_size = len;238 rdesc->rsvd0 = 0;239 rdesc->descriptor_overflow = 1; /* assume error */240 rdesc->buffer_overflow = 1; /* assume error */241 rdesc->last_seg = last;242 rdesc->first_seg = first;243 rdesc->result_size = EIP197_RD64_RESULT_SIZE;244 rdesc->rsvd1 = 0;245 rdesc->data_lo = lower_32_bits(data);246 rdesc->data_hi = upper_32_bits(data);247 248 /* Clear length in result token */249 rtoken->packet_length = 0;250 /* Assume errors - HW will clear if not the case */251 rtoken->error_code = 0x7fff;252 253 return rdesc;254}255