Skip to content

Commit c25be6c

Browse files
noctuellesgregkh
noctuelles
authored and
gregkh
committed
crypto: talitos - add chaining of arbitrary number of descriptor for the SEC1
commit f126384 upstream. The SEC1 hardware can process a chain of descriptors without host intervention. Only the hash implementation currently use this feature, but with a chain of at most 2 descriptors added in commit 37b5e88 ("crypto: talitos - chain in buffered data for ahash on SEC1"). Add supports for chaining an arbitrary number of descriptors in a chain. Adapt the ahash implementation to make it compatible. Cc: stable@vger.kernel.org Signed-off-by: Paul Louvel <paul.louvel@bootlin.com> Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au> Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
1 parent 3d1d944 commit c25be6c

2 files changed

Lines changed: 124 additions & 58 deletions

File tree

‎drivers/crypto/talitos.c‎

Lines changed: 122 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -273,7 +273,10 @@ static int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc,
273273
void *context, int error),
274274
void *context)
275275
{
276+
struct talitos_edesc *edesc = container_of(desc, struct talitos_edesc, desc);
276277
struct talitos_private *priv = dev_get_drvdata(dev);
278+
dma_addr_t dma_desc, prev_dma_desc;
279+
struct talitos_edesc *prev_edesc = NULL;
277280
struct talitos_request *request;
278281
unsigned long flags;
279282
int head;
@@ -292,10 +295,31 @@ static int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc,
292295

293296
/* map descriptor and save caller data */
294297
if (is_sec1) {
295-
desc->hdr1 = desc->hdr;
296-
request->dma_desc = dma_map_single(dev, &desc->hdr1,
298+
while (edesc) {
299+
edesc->desc.hdr1 = edesc->desc.hdr;
300+
301+
dma_desc = dma_map_single(dev, &edesc->desc.hdr1,
302+
TALITOS_DESC_SIZE,
303+
DMA_BIDIRECTIONAL);
304+
305+
if (!prev_edesc) {
306+
request->dma_desc = dma_desc;
307+
goto next;
308+
}
309+
310+
/* Chain in any previous descriptors. */
311+
312+
prev_edesc->desc.next_desc = cpu_to_be32(dma_desc);
313+
314+
dma_sync_single_for_device(dev, prev_dma_desc,
297315
TALITOS_DESC_SIZE,
298-
DMA_BIDIRECTIONAL);
316+
DMA_TO_DEVICE);
317+
318+
next:
319+
prev_edesc = edesc;
320+
prev_dma_desc = dma_desc;
321+
edesc = edesc->next_desc;
322+
}
299323
} else {
300324
request->dma_desc = dma_map_single(dev, desc,
301325
TALITOS_DESC_SIZE,
@@ -326,6 +350,7 @@ static __be32 get_request_hdr(struct device *dev,
326350
struct talitos_request *request, bool is_sec1)
327351
{
328352
struct talitos_edesc *edesc;
353+
dma_addr_t dma_desc;
329354

330355
if (!is_sec1) {
331356
dma_sync_single_for_cpu(dev, request->dma_desc,
@@ -334,19 +359,17 @@ static __be32 get_request_hdr(struct device *dev,
334359
return request->desc->hdr;
335360
}
336361

337-
if (!request->desc->next_desc) {
338-
dma_sync_single_for_cpu(dev, request->dma_desc,
339-
TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL);
340-
return request->desc->hdr1;
341-
} else {
342-
dma_sync_single_for_cpu(dev,
343-
be32_to_cpu(request->desc->next_desc),
344-
TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL);
345-
edesc = container_of(request->desc, struct talitos_edesc, desc);
346-
347-
return ((struct talitos_desc *)(edesc->buf + edesc->dma_len))
348-
->hdr1;
362+
edesc = container_of(request->desc, struct talitos_edesc, desc);
363+
dma_desc = request->dma_desc;
364+
while (edesc->next_desc) {
365+
dma_desc = be32_to_cpu(edesc->desc.next_desc);
366+
edesc = edesc->next_desc;
349367
}
368+
369+
dma_sync_single_for_cpu(dev, dma_desc, TALITOS_DESC_SIZE,
370+
DMA_BIDIRECTIONAL);
371+
372+
return edesc->desc.hdr1;
350373
}
351374

352375
/*
@@ -356,6 +379,7 @@ static void flush_channel(struct device *dev, int ch, int error, int reset_ch)
356379
{
357380
struct talitos_private *priv = dev_get_drvdata(dev);
358381
struct talitos_request *request, saved_req;
382+
struct talitos_edesc *edesc;
359383
unsigned long flags;
360384
int tail, status;
361385
bool is_sec1 = has_ftr_sec1(priv);
@@ -380,9 +404,22 @@ static void flush_channel(struct device *dev, int ch, int error, int reset_ch)
380404
else
381405
status = error;
382406

383-
dma_unmap_single(dev, request->dma_desc,
384-
TALITOS_DESC_SIZE,
385-
DMA_BIDIRECTIONAL);
407+
if (is_sec1) {
408+
dma_unmap_single(dev, request->dma_desc,
409+
TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL);
410+
edesc = container_of(request->desc,
411+
struct talitos_edesc, desc);
412+
while (edesc->next_desc) {
413+
dma_unmap_single(
414+
dev, be32_to_cpu(edesc->desc.next_desc),
415+
TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL);
416+
edesc = edesc->next_desc;
417+
}
418+
} else {
419+
dma_unmap_single(dev, request->dma_desc,
420+
TALITOS_DESC_SIZE,
421+
DMA_BIDIRECTIONAL);
422+
}
386423

387424
/* copy entries so we can call callback outside lock */
388425
saved_req.desc = request->desc;
@@ -477,8 +514,12 @@ DEF_TALITOS2_DONE(ch1_3, TALITOS2_ISR_CH_1_3_DONE)
477514
static __be32 current_desc_hdr(struct device *dev, int ch)
478515
{
479516
struct talitos_private *priv = dev_get_drvdata(dev);
517+
bool is_sec1 = has_ftr_sec1(priv);
518+
struct talitos_request *request;
519+
struct talitos_edesc *edesc;
480520
int tail, iter;
481521
dma_addr_t cur_desc;
522+
__be32 hdr = 0;
482523

483524
cur_desc = ((u64)in_be32(priv->chan[ch].reg + TALITOS_CDPR)) << 32;
484525
cur_desc |= in_be32(priv->chan[ch].reg + TALITOS_CDPR_LO);
@@ -489,27 +530,35 @@ static __be32 current_desc_hdr(struct device *dev, int ch)
489530
}
490531

491532
tail = priv->chan[ch].tail;
492-
493533
iter = tail;
494-
while (priv->chan[ch].fifo[iter].dma_desc != cur_desc &&
495-
priv->chan[ch].fifo[iter].desc->next_desc != cpu_to_be32(cur_desc)) {
496-
iter = (iter + 1) & (priv->fifo_len - 1);
497-
if (iter == tail) {
498-
dev_err(dev, "couldn't locate current descriptor\n");
499-
return 0;
534+
do {
535+
request = &priv->chan[ch].fifo[iter];
536+
537+
if (request->dma_desc == cur_desc) {
538+
hdr = request->desc->hdr;
539+
} else if (is_sec1) {
540+
edesc = container_of(request->desc,
541+
struct talitos_edesc, desc);
542+
while (edesc->next_desc) {
543+
if (edesc->desc.next_desc ==
544+
cpu_to_be32(cur_desc)) {
545+
hdr = edesc->next_desc->desc.hdr1;
546+
break;
547+
}
548+
edesc = edesc->next_desc;
549+
}
500550
}
501-
}
502551

503-
if (priv->chan[ch].fifo[iter].desc->next_desc == cpu_to_be32(cur_desc)) {
504-
struct talitos_edesc *edesc;
552+
if (hdr)
553+
break;
505554

506-
edesc = container_of(priv->chan[ch].fifo[iter].desc,
507-
struct talitos_edesc, desc);
508-
return ((struct talitos_desc *)
509-
(edesc->buf + edesc->dma_len))->hdr;
510-
}
555+
iter = (iter + 1) & (priv->fifo_len - 1);
556+
} while (iter != tail);
557+
558+
if (!hdr)
559+
dev_err(dev, "couldn't locate current descriptor\n");
511560

512-
return priv->chan[ch].fifo[iter].desc->hdr;
561+
return hdr;
513562
}
514563

515564
/*
@@ -1408,10 +1457,6 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev,
14081457
dma_len = 0;
14091458
}
14101459
alloc_len += icv_stashing ? authsize : 0;
1411-
1412-
/* if its a ahash, add space for a second desc next to the first one */
1413-
if (is_sec1 && !dst)
1414-
alloc_len += sizeof(struct talitos_desc);
14151460
alloc_len += ivsize;
14161461

14171462
edesc = kmalloc(ALIGN(alloc_len, dma_get_cache_alignment()), flags);
@@ -1427,6 +1472,7 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev,
14271472
edesc->dst_nents = dst_nents;
14281473
edesc->iv_dma = iv_dma;
14291474
edesc->dma_len = dma_len;
1475+
edesc->next_desc = NULL;
14301476
if (dma_len)
14311477
edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0],
14321478
edesc->dma_len,
@@ -1727,8 +1773,10 @@ static void common_nonsnoop_hash_unmap(struct device *dev,
17271773
struct talitos_private *priv = dev_get_drvdata(dev);
17281774
bool is_sec1 = has_ftr_sec1(priv);
17291775
struct talitos_desc *desc = &edesc->desc;
1730-
struct talitos_desc *desc2 = (struct talitos_desc *)
1731-
(edesc->buf + edesc->dma_len);
1776+
struct talitos_desc *desc2;
1777+
1778+
if (desc->next_desc)
1779+
desc2 = &edesc->next_desc->desc;
17321780

17331781
unmap_single_talitos_ptr(dev, &desc->ptr[5], DMA_FROM_DEVICE);
17341782
if (desc->next_desc &&
@@ -1756,10 +1804,17 @@ static void common_nonsnoop_hash_unmap(struct device *dev,
17561804
if (edesc->dma_len)
17571805
dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
17581806
DMA_BIDIRECTIONAL);
1807+
}
17591808

1760-
if (desc->next_desc)
1761-
dma_unmap_single(dev, be32_to_cpu(desc->next_desc),
1762-
TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL);
1809+
static void free_edesc_list_from(struct talitos_edesc *edesc)
1810+
{
1811+
struct talitos_edesc *next;
1812+
1813+
while (edesc) {
1814+
next = edesc->next_desc;
1815+
kfree(edesc);
1816+
edesc = next;
1817+
}
17631818
}
17641819

17651820
static void ahash_done(struct device *dev,
@@ -1778,7 +1833,7 @@ static void ahash_done(struct device *dev,
17781833
}
17791834
common_nonsnoop_hash_unmap(dev, edesc, areq);
17801835

1781-
kfree(edesc);
1836+
free_edesc_list_from(edesc);
17821837

17831838
if (err) {
17841839
ahash_request_complete(areq, err);
@@ -1894,14 +1949,23 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc,
18941949
talitos_handle_buggy_hash(ctx, edesc, &desc->ptr[3]);
18951950

18961951
if (is_sec1 && req_ctx->nbuf && length) {
1897-
struct talitos_desc *desc2 = (struct talitos_desc *)
1898-
(edesc->buf + edesc->dma_len);
1899-
dma_addr_t next_desc;
1952+
struct talitos_edesc *edesc2;
1953+
struct talitos_desc *desc2;
1954+
1955+
edesc2 = kzalloc(sizeof(*edesc2),
1956+
areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
1957+
GFP_KERNEL :
1958+
GFP_ATOMIC);
1959+
if (!edesc2) {
1960+
ret = -ENOMEM;
1961+
goto err;
1962+
}
1963+
edesc->next_desc = edesc2;
1964+
1965+
desc2 = &edesc2->desc;
19001966

1901-
memset(desc2, 0, sizeof(*desc2));
19021967
desc2->hdr = desc->hdr;
19031968
desc2->hdr &= ~DESC_HDR_MODE0_MDEU_INIT;
1904-
desc2->hdr1 = desc2->hdr;
19051969
desc->hdr &= ~DESC_HDR_MODE0_MDEU_PAD;
19061970
desc->hdr |= DESC_HDR_MODE0_MDEU_CONT;
19071971
desc->hdr &= ~DESC_HDR_DONE_NOTIFY;
@@ -1925,21 +1989,21 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc,
19251989
req_ctx->hw_context_size,
19261990
req_ctx->hw_context,
19271991
DMA_FROM_DEVICE);
1928-
1929-
next_desc = dma_map_single(dev, &desc2->hdr1, TALITOS_DESC_SIZE,
1930-
DMA_BIDIRECTIONAL);
1931-
desc->next_desc = cpu_to_be32(next_desc);
19321992
}
19331993

19341994
if (sync_needed)
19351995
dma_sync_single_for_device(dev, edesc->dma_link_tbl,
19361996
edesc->dma_len, DMA_BIDIRECTIONAL);
19371997

1938-
ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1939-
if (ret != -EINPROGRESS) {
1940-
common_nonsnoop_hash_unmap(dev, edesc, areq);
1941-
kfree(edesc);
1942-
}
1998+
ret = talitos_submit(dev, ctx->ch, desc, callback,
1999+
areq);
2000+
if (ret != -EINPROGRESS)
2001+
goto err;
2002+
2003+
return -EINPROGRESS;
2004+
err:
2005+
common_nonsnoop_hash_unmap(dev, edesc, areq);
2006+
kfree(edesc);
19432007
return ret;
19442008
}
19452009

‎drivers/crypto/talitos.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ struct talitos_desc {
4949
* @iv_dma: dma address of iv for checking continuity and link table
5050
* @dma_len: length of dma mapped link_tbl space
5151
* @dma_link_tbl: bus physical address of link_tbl/buf
52+
* @next_desc: next descriptor
5253
* @desc: h/w descriptor
5354
* @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1) (SEC2)
5455
* @buf: input and output buffeur (if {src,dst}_nents > 1) (SEC1)
@@ -63,6 +64,7 @@ struct talitos_edesc {
6364
dma_addr_t iv_dma;
6465
int dma_len;
6566
dma_addr_t dma_link_tbl;
67+
struct talitos_edesc *next_desc;
6668
struct talitos_desc desc;
6769
union {
6870
DECLARE_FLEX_ARRAY(struct talitos_ptr, link_tbl);

0 commit comments

Comments
 (0)